git.iko.soy/text
on master help sign in
swap
bb3d862177 ahead, 0 behind
49 files changed+1291−457
since splitsnapshots
unifiedsplit
.gitignore+1 .hgtags+14 Data/Text.hs+120−58 Data/Text/Array.hs+5−4 Data/Text/Encoding.hs+10−21 Data/Text/Encoding/Error.hs+7−1 Data/Text/Internal.hs+7−1 Data/Text/Internal/Builder.hs+19−16 Data/Text/Internal/Builder/Functions.hs+2−1 Data/Text/Internal/Builder/Int/Digits.hs+1−1 Data/Text/Internal/Builder/RealFloat/Functions.hs+28 Data/Text/Internal/Fusion.hs+34−24 Data/Text/Internal/Fusion/CaseMapping.hs+291−19 Data/Text/Internal/Fusion/Common.hs+59−72 Data/Text/Internal/Fusion/Types.hs+8−13 Data/Text/Internal/Read.hs+3−3 Data/Text/Lazy.hs+147−21 Data/Text/Lazy/Builder.hs+1−1 Data/Text/Lazy/Builder/Int.hs+19−14 Data/Text/Lazy/Encoding.hs+9−21 Data/Text/Lazy/Read.hs+4−2 Data/Text/Read.hs+1−1 Data/Text/Show.hs+89 benchmarks/haskell/Benchmarks.hs+1−1 benchmarks/haskell/Benchmarks/DecodeUtf8.hs+2−2 benchmarks/haskell/Benchmarks/FileRead.hs+11−11 benchmarks/haskell/Benchmarks/FoldLines.hs+2−2 benchmarks/haskell/Benchmarks/Programs/BigTable.hs+2−2 benchmarks/haskell/Benchmarks/Programs/Cut.hs+2−2 benchmarks/haskell/Benchmarks/Programs/Fold.hs+2−2 benchmarks/haskell/Benchmarks/Programs/Sort.hs+10−9 benchmarks/haskell/Benchmarks/Programs/StripTags.hs+5−5 benchmarks/haskell/Benchmarks/Programs/Throughput.hs+8−8 benchmarks/haskell/Benchmarks/Pure.hs+7 benchmarks/haskell/Benchmarks/ReadNumbers.hs+2−5 benchmarks/haskell/Benchmarks/Stream.hs+16−5 benchmarks/text-benchmarks.cabal+14−4 changelog.md+109 scripts/CaseFolding.hs+1−1 scripts/CaseMapping.hs+3−3 scripts/SpecialCasing.hs+1−1 tests/LiteralRuleTest.hs+21 tests/Makefile+6−1 tests/Tests/Properties.hs+57−9 tests/Tests/QuickCheckUtils.hs+73−82 tests/cabal.config−1 tests/literal-rule-test.sh+29 tests/text-tests.cabal+4−1 text.cabal+24−6
.gitignore+1
@@ -1,2 +1,3 @@
11dist
22cabal-dev
3+cabal.sandbox.config
.hgtags+14
@@ -50,3 +50,17 @@ d99cd091cdf71ce807a4255f6cc509c3154f51ea 0.11.3.1
5050fb7299f22449750c653ac64dee4bd63c13ea6e4c 1.1.1.1
5151eb170cb533b63534d166b5fd5344b36676adf140 1.1.1.2
5252e749e05f715d02445803f653cd3d97d5395d7c6f 1.1.1.3
53+b0dff8ca1142595bab489a768ca62e6a6805aac4 1.1.1.4
54+f8a60279237c1c28e094868740d9188dd94fcc9e 1.2.0.0
55+6231451979b2d33e28038b75f2dab19c9731f5c3 1.2.0.1
56+99a3503b4ad001fc1894e90286e96d7dea1e9195 1.2.0.2
57+47f138edf15576b9adaeed64784e30811099f202 1.2.0.3
58+0b064b0c69106bbcd13cb0099e72ae5fa2347bf0 1.2.0.4
59+8bbb3e222a0037035d893f731b0a23be449f7346 1.2.0.5
60+18c7f8e19872762b7e28ee1871e72f70ea54e3cd 1.2.0.6
61+42f751b87b4a187db93f595a8166d51746ff57d4 1.2.1.0
62+78227a84740b70dcd4cf7a65f3bcf603d87e48e2 1.2.1.1
63+40f268e6735749ef36f487e1d928eaa957bb431d 1.2.1.2
64+631fd055e27fa10983e17e9ad36d1b285b85c661 1.2.1.3
65+8221a68ed5ec37028357df9c6c25f52320e54ad9 1.2.2.0
66+ce1c5ac65eb9f5580114b4e380092b05a485e95d 1.2.2.1
Data/Text.hs+120−58
@@ -129,6 +129,7 @@ module Data.Text
129129 , drop
130130 , dropEnd
131131 , takeWhile
132+ , takeWhileEnd
132133 , dropWhile
133134 , dropWhileEnd
134135 , dropAround
@@ -191,15 +192,16 @@ module Data.Text
191192
192193 -- * Low level operations
193194 , copy
195+ , unpackCString#
194196 ) where
195197
196198import Prelude (Char, Bool(..), Int, Maybe(..), String,
197199 Eq(..), Ord(..), Ordering(..), (++),
198− Read(..), Show(..),
200+ Read(..),
199201 (&&), (||), (+), (-), (.), ($), ($!), (>>),
200202 not, return, otherwise, quot)
201203#if defined(HAVE_DEEPSEQ)
202−import Control.DeepSeq (NFData)
204+import Control.DeepSeq (NFData(rnf))
203205#endif
204206#if defined(ASSERTS)
205207import Control.Exception (assert)
@@ -211,13 +213,19 @@ import Control.Monad (foldM)
211213import Control.Monad.ST (ST)
212214import qualified Data.Text.Array as A
213215import qualified Data.List as L
216+import Data.Binary (Binary(get, put))
214217import Data.Monoid (Monoid(..))
218+#if MIN_VERSION_base(4,9,0)
219+import Data.Semigroup (Semigroup(..))
220+#endif
215221import Data.String (IsString(..))
216222import qualified Data.Text.Internal.Fusion as S
217223import qualified Data.Text.Internal.Fusion.Common as S
224+import Data.Text.Encoding (decodeUtf8', encodeUtf8)
218225import Data.Text.Internal.Fusion (stream, reverseStream, unstream)
219226import Data.Text.Internal.Private (span_)
220227import Data.Text.Internal (Text(..), empty, firstf, mul, safe, text)
228+import Data.Text.Show (singleton, unpack, unpackCString#)
221229import qualified Prelude as P
222230import Data.Text.Unsafe (Iter(..), iter, iter_, lengthWord16, reverseIter,
223231 reverseIter_, unsafeHead, unsafeTail)
@@ -230,15 +238,13 @@ import Data.ByteString (ByteString)
230238import qualified Data.Text.Lazy as L
231239import Data.Int (Int64)
232240#endif
233−#if __GLASGOW_HASKELL__ >= 702
234−import qualified GHC.CString as GHC
235−#else
236−import qualified GHC.Base as GHC
237−#endif
241+import GHC.Base (eqInt, neInt, gtInt, geInt, ltInt, leInt)
238242#if __GLASGOW_HASKELL__ >= 708
239243import qualified GHC.Exts as Exts
240244#endif
241−import GHC.Prim (Addr#)
245+#if MIN_VERSION_base(4,7,0)
246+import Text.Printf (PrintfArg, formatArg, formatString)
247+#endif
242248
243249-- $strict
244250--
@@ -324,15 +330,24 @@ instance Eq Text where
324330instance Ord Text where
325331 compare = compareText
326332
327−instance Show Text where
328− showsPrec p ps r = showsPrec p (unpack ps) r
329−
330333instance Read Text where
331334 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]
332335
336+#if MIN_VERSION_base(4,9,0)
337+-- Semigroup orphan instances for older GHCs are provided by
338+-- 'semigroups` package
339+
340+instance Semigroup Text where
341+ (<>) = append
342+#endif
343+
333344instance Monoid Text where
334345 mempty = empty
346+#if MIN_VERSION_base(4,9,0)
347+ mappend = (<>) -- future-proof definition
348+#else
335349 mappend = append
350+#endif
336351 mconcat = concat
337352
338353instance IsString Text where
@@ -346,9 +361,17 @@ instance Exts.IsList Text where
346361#endif
347362
348363#if defined(HAVE_DEEPSEQ)
349−instance NFData Text
364+instance NFData Text where rnf !_ = ()
350365#endif
351366
367+instance Binary Text where
368+ put t = put (encodeUtf8 t)
369+ get = do
370+ bs <- get
371+ case decodeUtf8' bs of
372+ P.Left exn -> P.fail (P.show exn)
373+ P.Right a -> P.return a
374+
352375-- | This instance preserves data abstraction at the cost of inefficiency.
353376-- We omit reflection services for the sake of data abstraction.
354377--
@@ -372,6 +395,12 @@ instance Data Text where
372395 _ -> P.error "gunfold"
373396 dataTypeOf _ = textDataType
374397
398+#if MIN_VERSION_base(4,7,0)
399+-- | Only defined for @base-4.7.0.0@ and later
400+instance PrintfArg Text where
401+ formatArg txt = formatString $ unpack txt
402+#endif
403+
375404packConstr :: Constr
376405packConstr = mkConstr textDataType "pack" [] Prefix
377406
@@ -401,30 +430,6 @@ pack :: String -> Text
401430pack = unstream . S.map safe . S.streamList
402431{-# INLINE [1] pack #-}
403432
404−-- | /O(n)/ Convert a 'Text' into a 'String'. Subject to fusion.
405−unpack :: Text -> String
406−unpack = S.unstreamList . stream
407−{-# INLINE [1] unpack #-}
408−
409−-- | /O(n)/ Convert a literal string into a Text. Subject to fusion.
410−unpackCString# :: Addr# -> Text
411−unpackCString# addr# = unstream (S.streamCString# addr#)
412−{-# NOINLINE unpackCString# #-}
413−
414−{-# RULES "TEXT literal" forall a.
415− unstream (S.map safe (S.streamList (GHC.unpackCString# a)))
416− = unpackCString# a #-}
417−
418−{-# RULES "TEXT literal UTF8" forall a.
419− unstream (S.map safe (S.streamList (GHC.unpackCStringUtf8# a)))
420− = unpackCString# a #-}
421−
422−-- | /O(1)/ Convert a character into a Text. Subject to fusion.
423−-- Performs replacement on invalid scalar values.
424−singleton :: Char -> Text
425−singleton = unstream . S.singleton . safe
426−{-# INLINE [1] singleton #-}
427−
428433-- -----------------------------------------------------------------------------
429434-- * Basic functions
430435
@@ -481,8 +486,8 @@ head t = S.head (stream t)
481486uncons :: Text -> Maybe (Char, Text)
482487uncons t@(Text arr off len)
483488 | len <= 0 = Nothing
484− | otherwise = Just (c, text arr (off+d) (len-d))
485− where Iter c d = iter t 0
489+ | otherwise = Just $ let !(Iter c d) = iter t 0
490+ in (c, text arr (off+d) (len-d))
486491{-# INLINE [1] uncons #-}
487492
488493-- | Lifted from Control.Arrow and specialized.
@@ -567,7 +572,9 @@ isSingleton = S.isSingleton . stream
567572-- Subject to fusion.
568573length :: Text -> Int
569574length t = S.length (stream t)
570−{-# INLINE length #-}
575+{-# INLINE [0] length #-}
576+-- length needs to be phased after the compareN/length rules otherwise
577+-- it may inline before the rules have an opportunity to fire.
571578
572579-- | /O(n)/ Compare the count of characters in a 'Text' to a number.
573580-- Subject to fusion.
@@ -586,32 +593,32 @@ compareLength t n = S.compareLengthI (stream t) n
586593
587594{-# RULES
588595"TEXT ==N/length -> compareLength/==EQ" [~1] forall t n.
589− (==) (length t) n = compareLength t n == EQ
596+ eqInt (length t) n = compareLength t n == EQ
590597 #-}
591598
592599{-# RULES
593600"TEXT /=N/length -> compareLength//=EQ" [~1] forall t n.
594− (/=) (length t) n = compareLength t n /= EQ
601+ neInt (length t) n = compareLength t n /= EQ
595602 #-}
596603
597604{-# RULES
598605"TEXT <N/length -> compareLength/==LT" [~1] forall t n.
599− (<) (length t) n = compareLength t n == LT
606+ ltInt (length t) n = compareLength t n == LT
600607 #-}
601608
602609{-# RULES
603610"TEXT <=N/length -> compareLength//=GT" [~1] forall t n.
604− (<=) (length t) n = compareLength t n /= GT
611+ leInt (length t) n = compareLength t n /= GT
605612 #-}
606613
607614{-# RULES
608615"TEXT >N/length -> compareLength/==GT" [~1] forall t n.
609− (>) (length t) n = compareLength t n == GT
616+ gtInt (length t) n = compareLength t n == GT
610617 #-}
611618
612619{-# RULES
613620"TEXT >=N/length -> compareLength//=LT" [~1] forall t n.
614− (>=) (length t) n = compareLength t n /= LT
621+ geInt (length t) n = compareLength t n /= LT
615622 #-}
616623
617624-- -----------------------------------------------------------------------------
@@ -642,11 +649,37 @@ reverse :: Text -> Text
642649reverse t = S.reverse (stream t)
643650{-# INLINE reverse #-}
644651
645−-- | /O(m+n)/ Replace every occurrence of one substring with another.
652+-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in
653+-- @haystack@ with @replacement@.
654+--
655+-- This function behaves as though it was defined as follows:
656+--
657+-- @
658+-- replace needle replacement haystack =
659+-- 'intercalate' replacement ('splitOn' needle haystack)
660+-- @
661+--
662+-- As this suggests, each occurrence is replaced exactly once. So if
663+-- @needle@ occurs in @replacement@, that occurrence will /not/ itself
664+-- be replaced recursively:
665+--
666+-- > replace "oo" "foo" "oo" == "foo"
667+--
668+-- In cases where several instances of @needle@ overlap, only the
669+-- first one will be replaced:
670+--
671+-- > replace "ofo" "bar" "ofofo" == "barfo"
646672--
647673-- In (unlikely) bad cases, this function's time complexity degrades
648674-- towards /O(n*m)/.
649−replace :: Text -> Text -> Text -> Text
675+replace :: Text
676+ -- ^ @needle@ to search for. If this string is empty, an
677+ -- error will occur.
678+ -> Text
679+ -- ^ @replacement@ to replace @needle@ with.
680+ -> Text
681+ -- ^ @haystack@ in which to search.
682+ -> Text
650683replace needle@(Text _ _ neeLen)
651684 (Text repArr repOff repLen)
652685 haystack@(Text hayArr hayOff hayLen)
@@ -708,7 +741,7 @@ replace needle@(Text _ _ neeLen)
708741-- instead of itself.
709742toCaseFold :: Text -> Text
710743toCaseFold t = unstream (S.toCaseFold (stream t))
711−{-# INLINE [0] toCaseFold #-}
744+{-# INLINE toCaseFold #-}
712745
713746-- | /O(n)/ Convert a string to lower case, using simple case
714747-- conversion. Subject to fusion.
@@ -882,13 +915,13 @@ concatMap f = concat . foldr ((:) . f) []
882915{-# INLINE concatMap #-}
883916
884917-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the
885−-- 'Text' @t@ satisifes the predicate @p@. Subject to fusion.
918+-- 'Text' @t@ satisfies the predicate @p@. Subject to fusion.
886919any :: (Char -> Bool) -> Text -> Bool
887920any p t = S.any p (stream t)
888921{-# INLINE any #-}
889922
890923-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the
891−-- 'Text' @t@ satisify the predicate @p@. Subject to fusion.
924+-- 'Text' @t@ satisfy the predicate @p@. Subject to fusion.
892925all :: (Char -> Bool) -> Text -> Bool
893926all p t = S.all p (stream t)
894927{-# INLINE all #-}
@@ -1119,6 +1152,27 @@ takeWhile p t@(Text arr off len) = loop 0
11191152 unstream (S.takeWhile p (stream t)) = takeWhile p t
11201153 #-}
11211154
1155+-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',
1156+-- returns the longest suffix (possibly empty) of elements that
1157+-- satisfy @p@. Subject to fusion.
1158+-- Examples:
1159+--
1160+-- > takeWhileEnd (=='o') "foo" == "oo"
1161+takeWhileEnd :: (Char -> Bool) -> Text -> Text
1162+takeWhileEnd p t@(Text arr off len) = loop (len-1) len
1163+ where loop !i !l | l <= 0 = t
1164+ | p c = loop (i+d) (l+d)
1165+ | otherwise = text arr (off+l) (len-l)
1166+ where (c,d) = reverseIter t i
1167+{-# INLINE [1] takeWhileEnd #-}
1168+
1169+{-# RULES
1170+"TEXT takeWhileEnd -> fused" [~1] forall p t.
1171+ takeWhileEnd p t = S.reverse (S.takeWhile p (S.reverseStream t))
1172+"TEXT takeWhileEnd -> unfused" [1] forall p t.
1173+ S.reverse (S.takeWhile p (S.reverseStream t)) = takeWhileEnd p t
1174+ #-}
1175+
11221176-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after
11231177-- 'takeWhile' @p@ @t@. Subject to fusion.
11241178dropWhile :: (Char -> Bool) -> Text -> Text
@@ -1137,8 +1191,9 @@ dropWhile p t@(Text arr off len) = loop 0 0
11371191 #-}
11381192
11391193-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after
1140−-- dropping characters that fail the predicate @p@ from the end of
1194+-- dropping characters that satisfy the predicate @p@ from the end of
11411195-- @t@. Subject to fusion.
1196+--
11421197-- Examples:
11431198--
11441199-- > dropWhileEnd (=='.') "foo..." == "foo"
@@ -1158,7 +1213,7 @@ dropWhileEnd p t@(Text arr off len) = loop (len-1) len
11581213 #-}
11591214
11601215-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after
1161−-- dropping characters that fail the predicate @p@ from both the
1216+-- dropping characters that satisfy the predicate @p@ from both the
11621217-- beginning and end of @t@. Subject to fusion.
11631218dropAround :: (Char -> Bool) -> Text -> Text
11641219dropAround p = dropWhile p . dropWhileEnd p
@@ -1253,9 +1308,9 @@ tails t | null t = [empty]
12531308-- copies to create substrings; they just construct new 'Text's that
12541309-- are slices of the original.
12551310
1256−-- | /O(m+n)/ Break a 'Text' into pieces separated by the first
1257−-- 'Text' argument, consuming the delimiter. An empty delimiter is
1258−-- invalid, and will cause an error to be raised.
1311+-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'
1312+-- argument (which cannot be empty), consuming the delimiter. An empty
1313+-- delimiter is invalid, and will cause an error to be raised.
12591314--
12601315-- Examples:
12611316--
@@ -1268,9 +1323,16 @@ tails t | null t = [empty]
12681323-- > intercalate s . splitOn s == id
12691324-- > splitOn (singleton c) == split (==c)
12701325--
1326+-- (Note: the string @s@ to split on above cannot be empty.)
1327+--
12711328-- In (unlikely) bad cases, this function's time complexity degrades
12721329-- towards /O(n*m)/.
1273−splitOn :: Text -> Text -> [Text]
1330+splitOn :: Text
1331+ -- ^ String to split on. If this string is empty, an error
1332+ -- will occur.
1333+ -> Text
1334+ -- ^ Input text.
1335+ -> [Text]
12741336splitOn pat@(Text _ _ l) src@(Text arr off len)
12751337 | l <= 0 = emptyError "splitOn"
12761338 | isSingleton pat = split (== unsafeHead pat) src
@@ -1478,7 +1540,7 @@ countChar c t = S.countChar c (stream t)
14781540-- equivalent to a pair of 'unpack' operations.
14791541zip :: Text -> Text -> [(Char,Char)]
14801542zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)
1481−{-# INLINE [0] zip #-}
1543+{-# INLINE zip #-}
14821544
14831545-- | /O(n)/ 'zipWith' generalises 'zip' by zipping with the function
14841546-- given as the first argument, instead of a tupling function.
@@ -1486,7 +1548,7 @@ zip a b = S.unstreamList $ S.zipWith (,) (stream a) (stream b)
14861548zipWith :: (Char -> Char -> Char) -> Text -> Text -> Text
14871549zipWith f t1 t2 = unstream (S.zipWith g (stream t1) (stream t2))
14881550 where g a b = safe (f a b)
1489−{-# INLINE [0] zipWith #-}
1551+{-# INLINE zipWith #-}
14901552
14911553-- | /O(n)/ Breaks a 'Text' up into a list of words, delimited by 'Char's
14921554-- representing white space.
Data/Text/Array.hs+5−4
@@ -20,7 +20,7 @@
2020-- > import qualified Data.Text.Array as A
2121--
2222-- The names in this module resemble those in the 'Data.Array' family
23−-- of modules, but are shorter due to the assumption of qualifid
23+-- of modules, but are shorter due to the assumption of qualified
2424-- naming.
2525module Data.Text.Array
2626 (
@@ -74,7 +74,7 @@ import Foreign.C.Types (CInt, CSize)
7474#endif
7575import GHC.Base (ByteArray#, MutableByteArray#, Int(..),
7676 indexWord16Array#, newByteArray#,
77− unsafeCoerce#, writeWord16Array#)
77+ unsafeFreezeByteArray#, writeWord16Array#)
7878import GHC.ST (ST(..), runST)
7979import GHC.Word (Word16(..))
8080import Prelude hiding (length, read)
@@ -130,8 +130,9 @@ array_size_error = error "Data.Text.Array.new: size overflow"
130130
131131-- | Freeze a mutable array. Do not mutate the 'MArray' afterwards!
132132unsafeFreeze :: MArray s -> ST s Array
133−unsafeFreeze MArray{..} = ST $ \s# ->
134− (# s#, Array (unsafeCoerce# maBA)
133+unsafeFreeze MArray{..} = ST $ \s1# ->
134+ case unsafeFreezeByteArray# maBA s1# of
135+ (# s2#, ba# #) -> (# s2#, Array ba#
135136#if defined(ASSERTS)
136137 maLen
137138#endif
Data/Text/Encoding.hs+10−21
@@ -55,13 +55,9 @@ module Data.Text.Encoding
5555 , encodeUtf32LE
5656 , encodeUtf32BE
5757
58−#if MIN_VERSION_bytestring(0,10,4)
5958 -- * Encoding Text using ByteString Builders
60− -- | /Note/ that these functions are only available if built against
61− -- @bytestring >= 0.10.4.0@.
6259 , encodeUtf8Builder
6360 , encodeUtf8BuilderEscaped
64−#endif
6561 ) where
6662
6763#if __GLASGOW_HASKELL__ >= 702
@@ -70,26 +66,17 @@ import Control.Monad.ST.Unsafe (unsafeIOToST, unsafeSTToIO)
7066import Control.Monad.ST (unsafeIOToST, unsafeSTToIO)
7167#endif
7268
73−#if MIN_VERSION_bytestring(0,10,4)
74−import Data.Bits ((.&.))
75−import Data.Text.Internal.Unsafe.Char (ord)
76−import qualified Data.ByteString.Builder as B
77−import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)
78−import qualified Data.ByteString.Builder.Prim as BP
79−import qualified Data.ByteString.Builder.Prim.Internal as BP
80−import qualified Data.Text.Internal.Encoding.Utf16 as U16
81−#endif
82−
8369import Control.Exception (evaluate, try)
8470import Control.Monad.ST (runST)
71+import Data.Bits ((.&.))
8572import Data.ByteString as B
8673import Data.ByteString.Internal as B hiding (c2w)
87−import Data.Text ()
8874import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
8975import Data.Text.Internal (Text(..), safe, text)
9076import Data.Text.Internal.Private (runText)
91−import Data.Text.Internal.Unsafe.Char (unsafeWrite)
77+import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
9278import Data.Text.Internal.Unsafe.Shift (shiftR)
79+import Data.Text.Show ()
9380import Data.Text.Unsafe (unsafeDupablePerformIO)
9481import Data.Word (Word8, Word32)
9582import Foreign.C.Types (CSize(..))
@@ -98,8 +85,13 @@ import Foreign.Marshal.Utils (with)
9885import Foreign.Ptr (Ptr, minusPtr, nullPtr, plusPtr)
9986import Foreign.Storable (Storable, peek, poke)
10087import GHC.Base (ByteArray#, MutableByteArray#)
88+import qualified Data.ByteString.Builder as B
89+import qualified Data.ByteString.Builder.Internal as B hiding (empty, append)
90+import qualified Data.ByteString.Builder.Prim as BP
91+import qualified Data.ByteString.Builder.Prim.Internal as BP
10192import qualified Data.Text.Array as A
10293import qualified Data.Text.Internal.Encoding.Fusion as E
94+import qualified Data.Text.Internal.Encoding.Utf16 as U16
10395import qualified Data.Text.Internal.Fusion as F
10496
10597#include "text_cbits.h"
@@ -267,6 +259,7 @@ streamDecodeUtf8With onErr = decodeChunk B.empty 0 0
267259 UTF8_REJECT -> do
268260 -- We encountered an encoding error
269261 x <- peek curPtr'
262+ poke statePtr 0
270263 case onErr desc (Just x) of
271264 Nothing -> loop $ curPtr' `plusPtr` 1
272265 Just c -> do
@@ -274,7 +267,6 @@ streamDecodeUtf8With onErr = decodeChunk B.empty 0 0
274267 w <- unsafeSTToIO $
275268 unsafeWrite dest (fromIntegral destOff) (safe c)
276269 poke destOffPtr (destOff + fromIntegral w)
277− poke statePtr 0
278270 loop $ curPtr' `plusPtr` 1
279271
280272 _ -> do
@@ -286,7 +278,7 @@ streamDecodeUtf8With onErr = decodeChunk B.empty 0 0
286278 return $! text arr 0 (fromIntegral n)
287279 lastPtr <- peek curPtrPtr
288280 let left = lastPtr `minusPtr` curPtr
289− undecoded = case state of
281+ !undecoded = case state of
290282 UTF8_ACCEPT -> B.empty
291283 _ -> B.append undecoded0 (B.drop left bs)
292284 return $ Some chunkText undecoded
@@ -315,8 +307,6 @@ decodeUtf8' :: ByteString -> Either UnicodeException Text
315307decodeUtf8' = unsafeDupablePerformIO . try . evaluate . decodeUtf8With strictDecode
316308{-# INLINE decodeUtf8' #-}
317309
318−#if MIN_VERSION_bytestring(0,10,4)
319−
320310-- | Encode text to a ByteString 'B.Builder' using UTF-8 encoding.
321311encodeUtf8Builder :: Text -> B.Builder
322312encodeUtf8Builder = encodeUtf8BuilderEscaped (BP.liftFixedToBounded BP.word8)
@@ -377,7 +367,6 @@ encodeUtf8BuilderEscaped be =
377367 outerLoop i (B.BufferRange op ope)
378368 where
379369 poke8 j v = poke (op `plusPtr` j) (fromIntegral v :: Word8)
380−#endif
381370
382371-- | Encode text using UTF-8 encoding.
383372encodeUtf8 :: Text -> ByteString
Data/Text/Encoding/Error.hs+7−1
@@ -1,5 +1,7 @@
11{-# LANGUAGE CPP, DeriveDataTypeable #-}
2−#if __GLASGOW_HASKELL__ >= 702
2+#if __GLASGOW_HASKELL__ >= 704
3+{-# LANGUAGE Safe #-}
4+#elif __GLASGOW_HASKELL__ >= 702
35{-# LANGUAGE Trustworthy #-}
46#endif
57-- |
@@ -66,6 +68,7 @@ type OnError a b = String -> Maybe a -> Maybe b
6668type OnDecodeError = OnError Word8 Char
6769
6870-- | A handler for an encoding error.
71+{-# DEPRECATED OnEncodeError "This exception is never used in practice, and will be removed." #-}
6972type OnEncodeError = OnError Char Word8
7073
7174-- | An exception type for representing Unicode encoding errors.
@@ -78,6 +81,8 @@ data UnicodeException =
7881 -- under the given encoding, or ran out of input in mid-encode.
7982 deriving (Eq, Typeable)
8083
84+{-# DEPRECATED EncodeError "This constructor is never used, and will be removed." #-}
85+
8186showUnicodeException :: UnicodeException -> String
8287showUnicodeException (DecodeError desc (Just w))
8388 = "Cannot decode byte '\\x" ++ showHex w ("': " ++ desc)
@@ -107,6 +112,7 @@ lenientDecode :: OnDecodeError
107112lenientDecode _ _ = Just '\xfffd'
108113
109114-- | Throw a 'UnicodeException' if encoding fails.
115+{-# DEPRECATED strictEncode "This function always throws an exception, and will be removed." #-}
110116strictEncode :: OnEncodeError
111117strictEncode desc c = throw (EncodeError desc c)
112118
Data/Text/Internal.hs+7−1
@@ -36,6 +36,7 @@ module Data.Text.Internal
3636 , safe
3737 -- * Code that must be here for accessibility
3838 , empty
39+ , empty_
3940 -- * Utilities
4041 , firstf
4142 -- * Checked multiplication
@@ -81,6 +82,11 @@ empty :: Text
8182empty = Text A.empty 0 0
8283{-# INLINE [1] empty #-}
8384
85+-- | A non-inlined version of 'empty'.
86+empty_ :: Text
87+empty_ = Text A.empty 0 0
88+{-# NOINLINE empty_ #-}
89+
8490-- | Construct a 'Text' without invisibly pinning its byte array in
8591-- memory if its length has dwindled to zero.
8692text :: A.Array -> Int -> Int -> Text
@@ -110,7 +116,7 @@ safe :: Char -> Char
110116safe c
111117 | ord c .&. 0x1ff800 /= 0xd800 = c
112118 | otherwise = '\xfffd'
113−{-# INLINE safe #-}
119+{-# INLINE [0] safe #-}
114120
115121-- | Apply a function to the first element of an optional pair.
116122firstf :: (a -> c) -> Maybe (a,b) -> Maybe (c,b)
Data/Text/Internal/Builder.hs+19−16
@@ -6,7 +6,7 @@
66-- Module : Data.Text.Internal.Builder
77-- Copyright : (c) 2013 Bryan O'Sullivan
88-- (c) 2010 Johan Tibell
9−-- License : BSD3-style (see LICENSE)
9+-- License : BSD-style (see LICENSE)
1010--
1111-- Maintainer : Johan Tibell <johan.tibell@gmail.com>
1212-- Stability : experimental
@@ -58,13 +58,14 @@ module Data.Text.Internal.Builder
5858 ) where
5959
6060import Control.Monad.ST (ST, runST)
61−import Data.Bits ((.&.))
6261import Data.Monoid (Monoid(..))
62+#if MIN_VERSION_base(4,9,0)
63+import Data.Semigroup (Semigroup(..))
64+#endif
6365import Data.Text.Internal (Text(..))
6466import Data.Text.Internal.Lazy (smallChunkSize)
6567import Data.Text.Unsafe (inlineInterleaveST)
66−import Data.Text.Internal.Unsafe.Char (ord, unsafeWrite)
67−import Data.Text.Internal.Unsafe.Shift (shiftR)
68+import Data.Text.Internal.Unsafe.Char (unsafeWrite)
6869import Prelude hiding (map, putChar)
6970
7071import qualified Data.String as String
@@ -91,12 +92,22 @@ newtype Builder = Builder {
9192 -> ST s [S.Text]
9293 }
9394
95+#if MIN_VERSION_base(4,9,0)
96+instance Semigroup Builder where
97+ (<>) = append
98+ {-# INLINE (<>) #-}
99+#endif
100+
94101instance Monoid Builder where
95102 mempty = empty
96103 {-# INLINE mempty #-}
104+#if MIN_VERSION_base(4,9,0)
105+ mappend = (<>) -- future-proof definition
106+#else
97107 mappend = append
108+#endif
98109 {-# INLINE mappend #-}
99− mconcat = foldr mappend mempty
110+ mconcat = foldr mappend Data.Monoid.mempty
100111 {-# INLINE mconcat #-}
101112
102113instance String.IsString Builder where
@@ -127,17 +138,7 @@ empty = Builder (\ k buf -> k buf)
127138-- * @'toLazyText' ('singleton' c) = 'L.singleton' c@
128139--
129140singleton :: Char -> Builder
130−singleton c = writeAtMost 2 $ \ marr o ->
131− if n < 0x10000
132− then A.unsafeWrite marr o (fromIntegral n) >> return 1
133− else do
134− A.unsafeWrite marr o lo
135− A.unsafeWrite marr (o+1) hi
136− return 2
137− where n = ord c
138− m = n - 0x10000
139− lo = fromIntegral $ (m `shiftR` 10) + 0xD800
140− hi = fromIntegral $ (m .&. 0x3FF) + 0xDC00
141+singleton c = writeAtMost 2 $ \ marr o -> unsafeWrite marr o c
141142{-# INLINE singleton #-}
142143
143144------------------------------------------------------------------------
@@ -241,6 +242,8 @@ flush = Builder $ \ k buf@(Buffer p o u l) ->
241242 !t = Text arr o u
242243 ts <- inlineInterleaveST (k b)
243244 return $! t : ts
245+{-# INLINE [1] flush #-}
246+-- defer inlining so that flush/flush rule may fire.
244247
245248------------------------------------------------------------------------
246249
Data/Text/Internal/Builder/Functions.hs+2−1
@@ -23,7 +23,8 @@ module Data.Text.Internal.Builder.Functions
2323
2424import Data.Monoid (mappend)
2525import Data.Text.Lazy.Builder (Builder)
26−import GHC.Base
26+import GHC.Base (chr#,ord#,(+#),Int(I#),Char(C#))
27+import Prelude ()
2728
2829-- | Unsafe conversion for decimal digits.
2930{-# INLINE i2d #-}
Data/Text/Internal/Builder/Int/Digits.hs+1−1
@@ -2,7 +2,7 @@
22
33-- Module: Data.Text.Internal.Builder.Int.Digits
44-- Copyright: (c) 2013 Bryan O'Sullivan
5−-- License: BSD3
5+-- License: BSD-style
66-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>
77-- Stability: experimental
88-- Portability: portable
Data/Text/Internal/Builder/RealFloat/Functions.hs+28
@@ -1,3 +1,5 @@
1+{-# LANGUAGE CPP #-}
2+
13-- |
24-- Module: Data.Text.Internal.Builder.RealFloat.Functions
35-- Copyright: (c) The University of Glasgow 1994-2002
@@ -13,6 +15,30 @@ module Data.Text.Internal.Builder.RealFloat.Functions
1315 ) where
1416
1517roundTo :: Int -> [Int] -> (Int,[Int])
18+
19+#if MIN_VERSION_base(4,6,0)
20+
21+roundTo d is =
22+ case f d True is of
23+ x@(0,_) -> x
24+ (1,xs) -> (1, 1:xs)
25+ _ -> error "roundTo: bad Value"
26+ where
27+ b2 = base `quot` 2
28+
29+ f n _ [] = (0, replicate n 0)
30+ f 0 e (x:xs) | x == b2 && e && all (== 0) xs = (0, []) -- Round to even when at exactly half the base
31+ | otherwise = (if x >= b2 then 1 else 0, [])
32+ f n _ (i:xs)
33+ | i' == base = (1,0:ds)
34+ | otherwise = (0,i':ds)
35+ where
36+ (c,ds) = f (n-1) (even i) xs
37+ i' = c + i
38+ base = 10
39+
40+#else
41+
1642roundTo d is =
1743 case f d is of
1844 x@(0,_) -> x
@@ -27,3 +53,5 @@ roundTo d is =
2753 where
2854 (c,ds) = f (n-1) xs
2955 i' = c + i
56+
57+#endif
Data/Text/Internal/Fusion.hs+34−24
@@ -98,25 +98,35 @@ reverseStream (Text arr off len) = Stream next (off+len-1) (betweenSize (len `sh
9898-- | /O(n)/ Convert a 'Stream Char' into a 'Text'.
9999unstream :: Stream Char -> Text
100100unstream (Stream next0 s0 len) = runText $ \done -> do
101− let mlen = upperBound 4 len
101+ -- Before encoding each char we perform a buffer realloc check assuming
102+ -- worst case encoding size of two 16-bit units for the char. Just add an
103+ -- extra space to the buffer so that we do not end up reallocating even when
104+ -- all the chars are encoded as single unit.
105+ let mlen = upperBound 4 len + 1
102106 arr0 <- A.new mlen
103− let outer arr top = loop
107+ let outer !arr !maxi = encode
104108 where
105− loop !s !i =
106− case next0 s of
107− Done -> done arr i
108− Skip s' -> loop s' i
109− Yield x s'
110− | j >= top -> {-# SCC "unstream/resize" #-} do
111− let top' = (top + 1) `shiftL` 1
112− arr' <- A.new top'
113− A.copyM arr' 0 arr 0 top
114− outer arr' top' s i
115− | otherwise -> do d <- unsafeWrite arr i x
116− loop s' (i+d)
117− where j | ord x < 0x10000 = i
118− | otherwise = i + 1
119− outer arr0 mlen s0 0
109+ -- keep the common case loop as small as possible
110+ encode !si !di =
111+ case next0 si of
112+ Done -> done arr di
113+ Skip si' -> encode si' di
114+ Yield c si'
115+ -- simply check for the worst case
116+ | maxi < di + 1 -> realloc si di
117+ | otherwise -> do
118+ n <- unsafeWrite arr di c
119+ encode si' (di + n)
120+
121+ -- keep uncommon case separate from the common case code
122+ {-# NOINLINE realloc #-}
123+ realloc !si !di = do
124+ let newlen = (maxi + 1) * 2
125+ arr' <- A.new newlen
126+ A.copyM arr' 0 arr 0 di
127+ outer arr' (newlen - 1) si di
128+
129+ outer arr0 (mlen - 1) s0 0
120130{-# INLINE [0] unstream #-}
121131{-# RULES "STREAM stream/unstream fusion" forall s. stream (unstream s) = s #-}
122132
@@ -166,15 +176,15 @@ reverse (Stream next s len0)
166176-- | /O(n)/ Perform the equivalent of 'scanr' over a list, only with
167177-- the input and result reversed.
168178reverseScanr :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char
169−reverseScanr f z0 (Stream next0 s0 len) = Stream next (S1 :*: z0 :*: s0) (len+1) -- HINT maybe too low
179+reverseScanr f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low
170180 where
171181 {-# INLINE next #-}
172− next (S1 :*: z :*: s) = Yield z (S2 :*: z :*: s)
173− next (S2 :*: z :*: s) = case next0 s of
174− Yield x s' -> let !x' = f x z
175− in Yield x' (S2 :*: x' :*: s')
176− Skip s' -> Skip (S2 :*: z :*: s')
177− Done -> Done
182+ next (Scan1 z s) = Yield z (Scan2 z s)
183+ next (Scan2 z s) = case next0 s of
184+ Yield x s' -> let !x' = f x z
185+ in Yield x' (Scan2 x' s')
186+ Skip s' -> Skip (Scan2 z s')
187+ Done -> Done
178188{-# INLINE reverseScanr #-}
179189
180190-- | /O(n)/ Like 'unfoldr', 'unfoldrN' builds a stream from a seed
Data/Text/Internal/Fusion/CaseMapping.hs+291−19
@@ -1,17 +1,17 @@
11{-# LANGUAGE Rank2Types #-}
22-- AUTOMATICALLY GENERATED - DO NOT EDIT
3−-- Generated by scripts/SpecialCasing.hs
4−-- CaseFolding-6.3.0.txt
5−-- Date: 2012-12-20, 22:14:35 GMT [MD]
6−-- SpecialCasing-6.3.0.txt
7−-- Date: 2013-05-08, 13:54:51 GMT [MD]
3+-- Generated by scripts/CaseMapping.hs
4+-- CaseFolding-8.0.0.txt
5+-- Date: 2015-01-13, 18:16:36 GMT [MD]
6+-- SpecialCasing-8.0.0.txt
7+-- Date: 2014-12-16, 23:08:04 GMT [MD]
88
99module Data.Text.Internal.Fusion.CaseMapping where
1010import Data.Char
1111import Data.Text.Internal.Fusion.Types
1212
1313upperMapping :: forall s. Char -> s -> Step (CC s) Char
14−{-# INLINE upperMapping #-}
14+{-# NOINLINE upperMapping #-}
1515-- LATIN SMALL LETTER SHARP S
1616upperMapping '\x00df' s = Yield '\x0053' (CC s '\x0053' '\x0000')
1717-- LATIN SMALL LIGATURE FF
@@ -218,12 +218,12 @@ upperMapping '\x1fc7' s = Yield '\x0397' (CC s '\x0342' '\x0399')
218218upperMapping '\x1ff7' s = Yield '\x03a9' (CC s '\x0342' '\x0399')
219219upperMapping c s = Yield (toUpper c) (CC s '\0' '\0')
220220lowerMapping :: forall s. Char -> s -> Step (CC s) Char
221−{-# INLINE lowerMapping #-}
221+{-# NOINLINE lowerMapping #-}
222222-- LATIN CAPITAL LETTER I WITH DOT ABOVE
223223lowerMapping '\x0130' s = Yield '\x0069' (CC s '\x0307' '\x0000')
224224lowerMapping c s = Yield (toLower c) (CC s '\0' '\0')
225225titleMapping :: forall s. Char -> s -> Step (CC s) Char
226−{-# INLINE titleMapping #-}
226+{-# NOINLINE titleMapping #-}
227227-- LATIN SMALL LETTER SHARP S
228228titleMapping '\x00df' s = Yield '\x0053' (CC s '\x0073' '\x0000')
229229-- LATIN SMALL LIGATURE FF
@@ -322,7 +322,7 @@ titleMapping '\x1fc7' s = Yield '\x0397' (CC s '\x0342' '\x0345')
322322titleMapping '\x1ff7' s = Yield '\x03a9' (CC s '\x0342' '\x0345')
323323titleMapping c s = Yield (toTitle c) (CC s '\0' '\0')
324324foldMapping :: forall s. Char -> s -> Step (CC s) Char
325−{-# INLINE foldMapping #-}
325+{-# NOINLINE foldMapping #-}
326326-- MICRO SIGN
327327foldMapping '\x00b5' s = Yield '\x03bc' (CC s '\x0000' '\x0000')
328328-- LATIN SMALL LETTER SHARP S
@@ -359,10 +359,18 @@ foldMapping '\x03f1' s = Yield '\x03c1' (CC s '\x0000' '\x0000')
359359foldMapping '\x03f5' s = Yield '\x03b5' (CC s '\x0000' '\x0000')
360360-- ARMENIAN SMALL LIGATURE ECH YIWN
361361foldMapping '\x0587' s = Yield '\x0565' (CC s '\x0582' '\x0000')
362−-- GEORGIAN CAPITAL LETTER YN
363−foldMapping '\x10c7' s = Yield '\x2d27' (CC s '\x0000' '\x0000')
364−-- GEORGIAN CAPITAL LETTER AEN
365−foldMapping '\x10cd' s = Yield '\x2d2d' (CC s '\x0000' '\x0000')
362+-- CHEROKEE SMALL LETTER YE
363+foldMapping '\x13f8' s = Yield '\x13f0' (CC s '\x0000' '\x0000')
364+-- CHEROKEE SMALL LETTER YI
365+foldMapping '\x13f9' s = Yield '\x13f1' (CC s '\x0000' '\x0000')
366+-- CHEROKEE SMALL LETTER YO
367+foldMapping '\x13fa' s = Yield '\x13f2' (CC s '\x0000' '\x0000')
368+-- CHEROKEE SMALL LETTER YU
369+foldMapping '\x13fb' s = Yield '\x13f3' (CC s '\x0000' '\x0000')
370+-- CHEROKEE SMALL LETTER YV
371+foldMapping '\x13fc' s = Yield '\x13f4' (CC s '\x0000' '\x0000')
372+-- CHEROKEE SMALL LETTER MV
373+foldMapping '\x13fd' s = Yield '\x13f5' (CC s '\x0000' '\x0000')
366374-- LATIN SMALL LETTER H WITH LINE BELOW
367375foldMapping '\x1e96' s = Yield '\x0068' (CC s '\x0331' '\x0000')
368376-- LATIN SMALL LETTER T WITH DIAERESIS
@@ -537,12 +545,174 @@ foldMapping '\x1ff6' s = Yield '\x03c9' (CC s '\x0342' '\x0000')
537545foldMapping '\x1ff7' s = Yield '\x03c9' (CC s '\x0342' '\x03b9')
538546-- GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
539547foldMapping '\x1ffc' s = Yield '\x03c9' (CC s '\x03b9' '\x0000')
540−-- COPTIC CAPITAL LETTER BOHAIRIC KHEI
541−foldMapping '\x2cf2' s = Yield '\x2cf3' (CC s '\x0000' '\x0000')
542−-- LATIN CAPITAL LETTER C WITH BAR
543−foldMapping '\xa792' s = Yield '\xa793' (CC s '\x0000' '\x0000')
544−-- LATIN CAPITAL LETTER H WITH HOOK
545−foldMapping '\xa7aa' s = Yield '\x0266' (CC s '\x0000' '\x0000')
548+-- LATIN CAPITAL LETTER J WITH CROSSED-TAIL
549+foldMapping '\xa7b2' s = Yield '\x029d' (CC s '\x0000' '\x0000')
550+-- LATIN CAPITAL LETTER CHI
551+foldMapping '\xa7b3' s = Yield '\xab53' (CC s '\x0000' '\x0000')
552+-- LATIN CAPITAL LETTER BETA
553+foldMapping '\xa7b4' s = Yield '\xa7b5' (CC s '\x0000' '\x0000')
554+-- LATIN CAPITAL LETTER OMEGA
555+foldMapping '\xa7b6' s = Yield '\xa7b7' (CC s '\x0000' '\x0000')
556+-- CHEROKEE SMALL LETTER A
557+foldMapping '\xab70' s = Yield '\x13a0' (CC s '\x0000' '\x0000')
558+-- CHEROKEE SMALL LETTER E
559+foldMapping '\xab71' s = Yield '\x13a1' (CC s '\x0000' '\x0000')
560+-- CHEROKEE SMALL LETTER I
561+foldMapping '\xab72' s = Yield '\x13a2' (CC s '\x0000' '\x0000')
562+-- CHEROKEE SMALL LETTER O
563+foldMapping '\xab73' s = Yield '\x13a3' (CC s '\x0000' '\x0000')
564+-- CHEROKEE SMALL LETTER U
565+foldMapping '\xab74' s = Yield '\x13a4' (CC s '\x0000' '\x0000')
566+-- CHEROKEE SMALL LETTER V
567+foldMapping '\xab75' s = Yield '\x13a5' (CC s '\x0000' '\x0000')
568+-- CHEROKEE SMALL LETTER GA
569+foldMapping '\xab76' s = Yield '\x13a6' (CC s '\x0000' '\x0000')
570+-- CHEROKEE SMALL LETTER KA
571+foldMapping '\xab77' s = Yield '\x13a7' (CC s '\x0000' '\x0000')
572+-- CHEROKEE SMALL LETTER GE
573+foldMapping '\xab78' s = Yield '\x13a8' (CC s '\x0000' '\x0000')
574+-- CHEROKEE SMALL LETTER GI
575+foldMapping '\xab79' s = Yield '\x13a9' (CC s '\x0000' '\x0000')
576+-- CHEROKEE SMALL LETTER GO
577+foldMapping '\xab7a' s = Yield '\x13aa' (CC s '\x0000' '\x0000')
578+-- CHEROKEE SMALL LETTER GU
579+foldMapping '\xab7b' s = Yield '\x13ab' (CC s '\x0000' '\x0000')
580+-- CHEROKEE SMALL LETTER GV
581+foldMapping '\xab7c' s = Yield '\x13ac' (CC s '\x0000' '\x0000')
582+-- CHEROKEE SMALL LETTER HA
583+foldMapping '\xab7d' s = Yield '\x13ad' (CC s '\x0000' '\x0000')
584+-- CHEROKEE SMALL LETTER HE
585+foldMapping '\xab7e' s = Yield '\x13ae' (CC s '\x0000' '\x0000')
586+-- CHEROKEE SMALL LETTER HI
587+foldMapping '\xab7f' s = Yield '\x13af' (CC s '\x0000' '\x0000')
588+-- CHEROKEE SMALL LETTER HO
589+foldMapping '\xab80' s = Yield '\x13b0' (CC s '\x0000' '\x0000')
590+-- CHEROKEE SMALL LETTER HU
591+foldMapping '\xab81' s = Yield '\x13b1' (CC s '\x0000' '\x0000')
592+-- CHEROKEE SMALL LETTER HV
593+foldMapping '\xab82' s = Yield '\x13b2' (CC s '\x0000' '\x0000')
594+-- CHEROKEE SMALL LETTER LA
595+foldMapping '\xab83' s = Yield '\x13b3' (CC s '\x0000' '\x0000')
596+-- CHEROKEE SMALL LETTER LE
597+foldMapping '\xab84' s = Yield '\x13b4' (CC s '\x0000' '\x0000')
598+-- CHEROKEE SMALL LETTER LI
599+foldMapping '\xab85' s = Yield '\x13b5' (CC s '\x0000' '\x0000')
600+-- CHEROKEE SMALL LETTER LO
601+foldMapping '\xab86' s = Yield '\x13b6' (CC s '\x0000' '\x0000')
602+-- CHEROKEE SMALL LETTER LU
603+foldMapping '\xab87' s = Yield '\x13b7' (CC s '\x0000' '\x0000')
604+-- CHEROKEE SMALL LETTER LV
605+foldMapping '\xab88' s = Yield '\x13b8' (CC s '\x0000' '\x0000')
606+-- CHEROKEE SMALL LETTER MA
607+foldMapping '\xab89' s = Yield '\x13b9' (CC s '\x0000' '\x0000')
608+-- CHEROKEE SMALL LETTER ME
609+foldMapping '\xab8a' s = Yield '\x13ba' (CC s '\x0000' '\x0000')
610+-- CHEROKEE SMALL LETTER MI
611+foldMapping '\xab8b' s = Yield '\x13bb' (CC s '\x0000' '\x0000')
612+-- CHEROKEE SMALL LETTER MO
613+foldMapping '\xab8c' s = Yield '\x13bc' (CC s '\x0000' '\x0000')
614+-- CHEROKEE SMALL LETTER MU
615+foldMapping '\xab8d' s = Yield '\x13bd' (CC s '\x0000' '\x0000')
616+-- CHEROKEE SMALL LETTER NA
617+foldMapping '\xab8e' s = Yield '\x13be' (CC s '\x0000' '\x0000')
618+-- CHEROKEE SMALL LETTER HNA
619+foldMapping '\xab8f' s = Yield '\x13bf' (CC s '\x0000' '\x0000')
620+-- CHEROKEE SMALL LETTER NAH
621+foldMapping '\xab90' s = Yield '\x13c0' (CC s '\x0000' '\x0000')
622+-- CHEROKEE SMALL LETTER NE
623+foldMapping '\xab91' s = Yield '\x13c1' (CC s '\x0000' '\x0000')
624+-- CHEROKEE SMALL LETTER NI
625+foldMapping '\xab92' s = Yield '\x13c2' (CC s '\x0000' '\x0000')
626+-- CHEROKEE SMALL LETTER NO
627+foldMapping '\xab93' s = Yield '\x13c3' (CC s '\x0000' '\x0000')
628+-- CHEROKEE SMALL LETTER NU
629+foldMapping '\xab94' s = Yield '\x13c4' (CC s '\x0000' '\x0000')
630+-- CHEROKEE SMALL LETTER NV
631+foldMapping '\xab95' s = Yield '\x13c5' (CC s '\x0000' '\x0000')
632+-- CHEROKEE SMALL LETTER QUA
633+foldMapping '\xab96' s = Yield '\x13c6' (CC s '\x0000' '\x0000')
634+-- CHEROKEE SMALL LETTER QUE
635+foldMapping '\xab97' s = Yield '\x13c7' (CC s '\x0000' '\x0000')
636+-- CHEROKEE SMALL LETTER QUI
637+foldMapping '\xab98' s = Yield '\x13c8' (CC s '\x0000' '\x0000')
638+-- CHEROKEE SMALL LETTER QUO
639+foldMapping '\xab99' s = Yield '\x13c9' (CC s '\x0000' '\x0000')
640+-- CHEROKEE SMALL LETTER QUU
641+foldMapping '\xab9a' s = Yield '\x13ca' (CC s '\x0000' '\x0000')
642+-- CHEROKEE SMALL LETTER QUV
643+foldMapping '\xab9b' s = Yield '\x13cb' (CC s '\x0000' '\x0000')
644+-- CHEROKEE SMALL LETTER SA
645+foldMapping '\xab9c' s = Yield '\x13cc' (CC s '\x0000' '\x0000')
646+-- CHEROKEE SMALL LETTER S
647+foldMapping '\xab9d' s = Yield '\x13cd' (CC s '\x0000' '\x0000')
648+-- CHEROKEE SMALL LETTER SE
649+foldMapping '\xab9e' s = Yield '\x13ce' (CC s '\x0000' '\x0000')
650+-- CHEROKEE SMALL LETTER SI
651+foldMapping '\xab9f' s = Yield '\x13cf' (CC s '\x0000' '\x0000')
652+-- CHEROKEE SMALL LETTER SO
653+foldMapping '\xaba0' s = Yield '\x13d0' (CC s '\x0000' '\x0000')
654+-- CHEROKEE SMALL LETTER SU
655+foldMapping '\xaba1' s = Yield '\x13d1' (CC s '\x0000' '\x0000')
656+-- CHEROKEE SMALL LETTER SV
657+foldMapping '\xaba2' s = Yield '\x13d2' (CC s '\x0000' '\x0000')
658+-- CHEROKEE SMALL LETTER DA
659+foldMapping '\xaba3' s = Yield '\x13d3' (CC s '\x0000' '\x0000')
660+-- CHEROKEE SMALL LETTER TA
661+foldMapping '\xaba4' s = Yield '\x13d4' (CC s '\x0000' '\x0000')
662+-- CHEROKEE SMALL LETTER DE
663+foldMapping '\xaba5' s = Yield '\x13d5' (CC s '\x0000' '\x0000')
664+-- CHEROKEE SMALL LETTER TE
665+foldMapping '\xaba6' s = Yield '\x13d6' (CC s '\x0000' '\x0000')
666+-- CHEROKEE SMALL LETTER DI
667+foldMapping '\xaba7' s = Yield '\x13d7' (CC s '\x0000' '\x0000')
668+-- CHEROKEE SMALL LETTER TI
669+foldMapping '\xaba8' s = Yield '\x13d8' (CC s '\x0000' '\x0000')
670+-- CHEROKEE SMALL LETTER DO
671+foldMapping '\xaba9' s = Yield '\x13d9' (CC s '\x0000' '\x0000')
672+-- CHEROKEE SMALL LETTER DU
673+foldMapping '\xabaa' s = Yield '\x13da' (CC s '\x0000' '\x0000')
674+-- CHEROKEE SMALL LETTER DV
675+foldMapping '\xabab' s = Yield '\x13db' (CC s '\x0000' '\x0000')
676+-- CHEROKEE SMALL LETTER DLA
677+foldMapping '\xabac' s = Yield '\x13dc' (CC s '\x0000' '\x0000')
678+-- CHEROKEE SMALL LETTER TLA
679+foldMapping '\xabad' s = Yield '\x13dd' (CC s '\x0000' '\x0000')
680+-- CHEROKEE SMALL LETTER TLE
681+foldMapping '\xabae' s = Yield '\x13de' (CC s '\x0000' '\x0000')
682+-- CHEROKEE SMALL LETTER TLI
683+foldMapping '\xabaf' s = Yield '\x13df' (CC s '\x0000' '\x0000')
684+-- CHEROKEE SMALL LETTER TLO
685+foldMapping '\xabb0' s = Yield '\x13e0' (CC s '\x0000' '\x0000')
686+-- CHEROKEE SMALL LETTER TLU
687+foldMapping '\xabb1' s = Yield '\x13e1' (CC s '\x0000' '\x0000')
688+-- CHEROKEE SMALL LETTER TLV
689+foldMapping '\xabb2' s = Yield '\x13e2' (CC s '\x0000' '\x0000')
690+-- CHEROKEE SMALL LETTER TSA
691+foldMapping '\xabb3' s = Yield '\x13e3' (CC s '\x0000' '\x0000')
692+-- CHEROKEE SMALL LETTER TSE
693+foldMapping '\xabb4' s = Yield '\x13e4' (CC s '\x0000' '\x0000')
694+-- CHEROKEE SMALL LETTER TSI
695+foldMapping '\xabb5' s = Yield '\x13e5' (CC s '\x0000' '\x0000')
696+-- CHEROKEE SMALL LETTER TSO
697+foldMapping '\xabb6' s = Yield '\x13e6' (CC s '\x0000' '\x0000')
698+-- CHEROKEE SMALL LETTER TSU
699+foldMapping '\xabb7' s = Yield '\x13e7' (CC s '\x0000' '\x0000')
700+-- CHEROKEE SMALL LETTER TSV
701+foldMapping '\xabb8' s = Yield '\x13e8' (CC s '\x0000' '\x0000')
702+-- CHEROKEE SMALL LETTER WA
703+foldMapping '\xabb9' s = Yield '\x13e9' (CC s '\x0000' '\x0000')
704+-- CHEROKEE SMALL LETTER WE
705+foldMapping '\xabba' s = Yield '\x13ea' (CC s '\x0000' '\x0000')
706+-- CHEROKEE SMALL LETTER WI
707+foldMapping '\xabbb' s = Yield '\x13eb' (CC s '\x0000' '\x0000')
708+-- CHEROKEE SMALL LETTER WO
709+foldMapping '\xabbc' s = Yield '\x13ec' (CC s '\x0000' '\x0000')
710+-- CHEROKEE SMALL LETTER WU
711+foldMapping '\xabbd' s = Yield '\x13ed' (CC s '\x0000' '\x0000')
712+-- CHEROKEE SMALL LETTER WV
713+foldMapping '\xabbe' s = Yield '\x13ee' (CC s '\x0000' '\x0000')
714+-- CHEROKEE SMALL LETTER YA
715+foldMapping '\xabbf' s = Yield '\x13ef' (CC s '\x0000' '\x0000')
546716-- LATIN SMALL LIGATURE FF
547717foldMapping '\xfb00' s = Yield '\x0066' (CC s '\x0066' '\x0000')
548718-- LATIN SMALL LIGATURE FI
@@ -567,4 +737,106 @@ foldMapping '\xfb15' s = Yield '\x0574' (CC s '\x056b' '\x0000')
567737foldMapping '\xfb16' s = Yield '\x057e' (CC s '\x0576' '\x0000')
568738-- ARMENIAN SMALL LIGATURE MEN XEH
569739foldMapping '\xfb17' s = Yield '\x0574' (CC s '\x056d' '\x0000')
740+-- OLD HUNGARIAN CAPITAL LETTER A
741+foldMapping '\x10c80' s = Yield '\x10cc0' (CC s '\x0000' '\x0000')
742+-- OLD HUNGARIAN CAPITAL LETTER AA
743+foldMapping '\x10c81' s = Yield '\x10cc1' (CC s '\x0000' '\x0000')
744+-- OLD HUNGARIAN CAPITAL LETTER EB
745+foldMapping '\x10c82' s = Yield '\x10cc2' (CC s '\x0000' '\x0000')
746+-- OLD HUNGARIAN CAPITAL LETTER AMB
747+foldMapping '\x10c83' s = Yield '\x10cc3' (CC s '\x0000' '\x0000')
748+-- OLD HUNGARIAN CAPITAL LETTER EC
749+foldMapping '\x10c84' s = Yield '\x10cc4' (CC s '\x0000' '\x0000')
750+-- OLD HUNGARIAN CAPITAL LETTER ENC
751+foldMapping '\x10c85' s = Yield '\x10cc5' (CC s '\x0000' '\x0000')
752+-- OLD HUNGARIAN CAPITAL LETTER ECS
753+foldMapping '\x10c86' s = Yield '\x10cc6' (CC s '\x0000' '\x0000')
754+-- OLD HUNGARIAN CAPITAL LETTER ED
755+foldMapping '\x10c87' s = Yield '\x10cc7' (CC s '\x0000' '\x0000')
756+-- OLD HUNGARIAN CAPITAL LETTER AND
757+foldMapping '\x10c88' s = Yield '\x10cc8' (CC s '\x0000' '\x0000')
758+-- OLD HUNGARIAN CAPITAL LETTER E
759+foldMapping '\x10c89' s = Yield '\x10cc9' (CC s '\x0000' '\x0000')
760+-- OLD HUNGARIAN CAPITAL LETTER CLOSE E
761+foldMapping '\x10c8a' s = Yield '\x10cca' (CC s '\x0000' '\x0000')
762+-- OLD HUNGARIAN CAPITAL LETTER EE
763+foldMapping '\x10c8b' s = Yield '\x10ccb' (CC s '\x0000' '\x0000')
764+-- OLD HUNGARIAN CAPITAL LETTER EF
765+foldMapping '\x10c8c' s = Yield '\x10ccc' (CC s '\x0000' '\x0000')
766+-- OLD HUNGARIAN CAPITAL LETTER EG
767+foldMapping '\x10c8d' s = Yield '\x10ccd' (CC s '\x0000' '\x0000')
768+-- OLD HUNGARIAN CAPITAL LETTER EGY
769+foldMapping '\x10c8e' s = Yield '\x10cce' (CC s '\x0000' '\x0000')
770+-- OLD HUNGARIAN CAPITAL LETTER EH
771+foldMapping '\x10c8f' s = Yield '\x10ccf' (CC s '\x0000' '\x0000')
772+-- OLD HUNGARIAN CAPITAL LETTER I
773+foldMapping '\x10c90' s = Yield '\x10cd0' (CC s '\x0000' '\x0000')
774+-- OLD HUNGARIAN CAPITAL LETTER II
775+foldMapping '\x10c91' s = Yield '\x10cd1' (CC s '\x0000' '\x0000')
776+-- OLD HUNGARIAN CAPITAL LETTER EJ
777+foldMapping '\x10c92' s = Yield '\x10cd2' (CC s '\x0000' '\x0000')
778+-- OLD HUNGARIAN CAPITAL LETTER EK
779+foldMapping '\x10c93' s = Yield '\x10cd3' (CC s '\x0000' '\x0000')
780+-- OLD HUNGARIAN CAPITAL LETTER AK
781+foldMapping '\x10c94' s = Yield '\x10cd4' (CC s '\x0000' '\x0000')
782+-- OLD HUNGARIAN CAPITAL LETTER UNK
783+foldMapping '\x10c95' s = Yield '\x10cd5' (CC s '\x0000' '\x0000')
784+-- OLD HUNGARIAN CAPITAL LETTER EL
785+foldMapping '\x10c96' s = Yield '\x10cd6' (CC s '\x0000' '\x0000')
786+-- OLD HUNGARIAN CAPITAL LETTER ELY
787+foldMapping '\x10c97' s = Yield '\x10cd7' (CC s '\x0000' '\x0000')
788+-- OLD HUNGARIAN CAPITAL LETTER EM
789+foldMapping '\x10c98' s = Yield '\x10cd8' (CC s '\x0000' '\x0000')
790+-- OLD HUNGARIAN CAPITAL LETTER EN
791+foldMapping '\x10c99' s = Yield '\x10cd9' (CC s '\x0000' '\x0000')
792+-- OLD HUNGARIAN CAPITAL LETTER ENY
793+foldMapping '\x10c9a' s = Yield '\x10cda' (CC s '\x0000' '\x0000')
794+-- OLD HUNGARIAN CAPITAL LETTER O
795+foldMapping '\x10c9b' s = Yield '\x10cdb' (CC s '\x0000' '\x0000')
796+-- OLD HUNGARIAN CAPITAL LETTER OO
797+foldMapping '\x10c9c' s = Yield '\x10cdc' (CC s '\x0000' '\x0000')
798+-- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE
799+foldMapping '\x10c9d' s = Yield '\x10cdd' (CC s '\x0000' '\x0000')
800+-- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE
801+foldMapping '\x10c9e' s = Yield '\x10cde' (CC s '\x0000' '\x0000')
802+-- OLD HUNGARIAN CAPITAL LETTER OEE
803+foldMapping '\x10c9f' s = Yield '\x10cdf' (CC s '\x0000' '\x0000')
804+-- OLD HUNGARIAN CAPITAL LETTER EP
805+foldMapping '\x10ca0' s = Yield '\x10ce0' (CC s '\x0000' '\x0000')
806+-- OLD HUNGARIAN CAPITAL LETTER EMP
807+foldMapping '\x10ca1' s = Yield '\x10ce1' (CC s '\x0000' '\x0000')
808+-- OLD HUNGARIAN CAPITAL LETTER ER
809+foldMapping '\x10ca2' s = Yield '\x10ce2' (CC s '\x0000' '\x0000')
810+-- OLD HUNGARIAN CAPITAL LETTER SHORT ER
811+foldMapping '\x10ca3' s = Yield '\x10ce3' (CC s '\x0000' '\x0000')
812+-- OLD HUNGARIAN CAPITAL LETTER ES
813+foldMapping '\x10ca4' s = Yield '\x10ce4' (CC s '\x0000' '\x0000')
814+-- OLD HUNGARIAN CAPITAL LETTER ESZ
815+foldMapping '\x10ca5' s = Yield '\x10ce5' (CC s '\x0000' '\x0000')
816+-- OLD HUNGARIAN CAPITAL LETTER ET
817+foldMapping '\x10ca6' s = Yield '\x10ce6' (CC s '\x0000' '\x0000')
818+-- OLD HUNGARIAN CAPITAL LETTER ENT
819+foldMapping '\x10ca7' s = Yield '\x10ce7' (CC s '\x0000' '\x0000')
820+-- OLD HUNGARIAN CAPITAL LETTER ETY
821+foldMapping '\x10ca8' s = Yield '\x10ce8' (CC s '\x0000' '\x0000')
822+-- OLD HUNGARIAN CAPITAL LETTER ECH
823+foldMapping '\x10ca9' s = Yield '\x10ce9' (CC s '\x0000' '\x0000')
824+-- OLD HUNGARIAN CAPITAL LETTER U
825+foldMapping '\x10caa' s = Yield '\x10cea' (CC s '\x0000' '\x0000')
826+-- OLD HUNGARIAN CAPITAL LETTER UU
827+foldMapping '\x10cab' s = Yield '\x10ceb' (CC s '\x0000' '\x0000')
828+-- OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE
829+foldMapping '\x10cac' s = Yield '\x10cec' (CC s '\x0000' '\x0000')
830+-- OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE
831+foldMapping '\x10cad' s = Yield '\x10ced' (CC s '\x0000' '\x0000')
832+-- OLD HUNGARIAN CAPITAL LETTER EV
833+foldMapping '\x10cae' s = Yield '\x10cee' (CC s '\x0000' '\x0000')
834+-- OLD HUNGARIAN CAPITAL LETTER EZ
835+foldMapping '\x10caf' s = Yield '\x10cef' (CC s '\x0000' '\x0000')
836+-- OLD HUNGARIAN CAPITAL LETTER EZS
837+foldMapping '\x10cb0' s = Yield '\x10cf0' (CC s '\x0000' '\x0000')
838+-- OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN
839+foldMapping '\x10cb1' s = Yield '\x10cf1' (CC s '\x0000' '\x0000')
840+-- OLD HUNGARIAN CAPITAL LETTER US
841+foldMapping '\x10cb2' s = Yield '\x10cf2' (CC s '\x0000' '\x0000')
570842foldMapping c s = Yield (toLower c) (CC s '\0' '\0')
Data/Text/Internal/Fusion/Common.hs+59−72
@@ -71,9 +71,6 @@ module Data.Text.Internal.Fusion.Common
7171 -- ** Scans
7272 , scanl
7373
74− -- ** Accumulating maps
75− -- , mapAccumL
76−
7774 -- ** Generation and unfolding
7875 , replicateCharI
7976 , replicateI
@@ -123,7 +120,7 @@ singleton :: Char -> Stream Char
123120singleton c = Stream next False 1
124121 where next False = Yield c True
125122 next True = Done
126−{-# INLINE singleton #-}
123+{-# INLINE [0] singleton #-}
127124
128125streamList :: [a] -> Stream a
129126{-# INLINE [0] streamList #-}
@@ -187,6 +184,9 @@ cons !w (Stream next0 s0 len) = Stream next (C1 s0) (len+1)
187184 Yield x s' -> Yield x (C0 s')
188185{-# INLINE [0] cons #-}
189186
187+data Snoc a = N
188+ | J !a
189+
190190-- | /O(n)/ Adds a character to the end of a stream.
191191snoc :: Stream Char -> Char -> Stream Char
192192snoc (Stream next0 xs0 len) w = Stream next (J xs0) (len+1)
@@ -473,17 +473,20 @@ toTitle (Stream next0 s0 len) = Stream next (CC (False :*: s0) '\0' '\0') len
473473 next (CC s a b) = Yield a (CC s b '\0')
474474{-# INLINE [0] toTitle #-}
475475
476+data Justify i s = Just1 !i !s
477+ | Just2 !i !s
478+
476479justifyLeftI :: Integral a => a -> Char -> Stream Char -> Stream Char
477480justifyLeftI k c (Stream next0 s0 len) =
478− Stream next (s0 :*: S1 :*: 0) (larger (fromIntegral k) len)
481+ Stream next (Just1 0 s0) (larger (fromIntegral k) len)
479482 where
480− next (s :*: S1 :*: n) =
483+ next (Just1 n s) =
481484 case next0 s of
482− Done -> next (s :*: S2 :*: n)
483− Skip s' -> Skip (s' :*: S1 :*: n)
484− Yield x s' -> Yield x (s' :*: S1 :*: n+1)
485− next (s :*: S2 :*: n)
486− | n < k = Yield c (s :*: S2 :*: n+1)
485+ Done -> next (Just2 n s)
486+ Skip s' -> Skip (Just1 n s')
487+ Yield x s' -> Yield x (Just1 (n+1) s')
488+ next (Just2 n s)
489+ | n < k = Yield c (Just2 (n+1) s)
487490 | otherwise = Done
488491 {-# INLINE next #-}
489492{-# INLINE [0] justifyLeftI #-}
@@ -590,7 +593,7 @@ concatMap f = foldr (append . f) empty
590593{-# INLINE [0] concatMap #-}
591594
592595-- | /O(n)/ any @p @xs determines if any character in the stream
593−-- @xs@ satisifes the predicate @p@.
596+-- @xs@ satisfies the predicate @p@.
594597any :: (Char -> Bool) -> Stream Char -> Bool
595598any p (Stream next0 s0 _len) = loop_any s0
596599 where
@@ -602,7 +605,7 @@ any p (Stream next0 s0 _len) = loop_any s0
602605{-# INLINE [0] any #-}
603606
604607-- | /O(n)/ all @p @xs determines if all characters in the 'Text'
605−-- @xs@ satisify the predicate @p@.
608+-- @xs@ satisfy the predicate @p@.
606609all :: (Char -> Bool) -> Stream Char -> Bool
607610all p (Stream next0 s0 _len) = loop_all s0
608611 where
@@ -651,50 +654,27 @@ minimum (Stream next0 s0 _len) = loop0_minimum s0
651654-- * Building streams
652655
653656scanl :: (Char -> Char -> Char) -> Char -> Stream Char -> Stream Char
654−scanl f z0 (Stream next0 s0 len) = Stream next (S1 :*: z0 :*: s0) (len+1) -- HINT maybe too low
657+scanl f z0 (Stream next0 s0 len) = Stream next (Scan1 z0 s0) (len+1) -- HINT maybe too low
655658 where
656659 {-# INLINE next #-}
657− next (S1 :*: z :*: s) = Yield z (S2 :*: z :*: s)
658− next (S2 :*: z :*: s) = case next0 s of
659− Yield x s' -> let !x' = f z x
660− in Yield x' (S2 :*: x' :*: s')
661− Skip s' -> Skip (S2 :*: z :*: s')
662− Done -> Done
660+ next (Scan1 z s) = Yield z (Scan2 z s)
661+ next (Scan2 z s) = case next0 s of
662+ Yield x s' -> let !x' = f z x
663+ in Yield x' (Scan2 x' s')
664+ Skip s' -> Skip (Scan2 z s')
665+ Done -> Done
663666{-# INLINE [0] scanl #-}
664667
665−-- -----------------------------------------------------------------------------
666−-- ** Accumulating maps
667−
668−{-
669−-- | /O(n)/ Like a combination of 'map' and 'foldl'. Applies a
670−-- function to each element of a stream, passing an accumulating
671−-- parameter from left to right, and returns a final stream.
672−--
673−-- /Note/: Unlike the version over lists, this function does not
674−-- return a final value for the accumulator, because the nature of
675−-- streams precludes it.
676−mapAccumL :: (a -> b -> (a,b)) -> a -> Stream b -> Stream b
677−mapAccumL f z0 (Stream next0 s0 len) = Stream next (s0 :*: z0) len -- HINT depends on f
678− where
679− {-# INLINE next #-}
680− next (s :*: z) = case next0 s of
681− Yield x s' -> let (z',y) = f z x
682− in Yield y (s' :*: z')
683− Skip s' -> Skip (s' :*: z)
684− Done -> Done
685−{-# INLINE [0] mapAccumL #-}
686−-}
687−
688668-- -----------------------------------------------------------------------------
689669-- ** Generating and unfolding streams
690670
691671replicateCharI :: Integral a => a -> Char -> Stream Char
692−replicateCharI n c
672+replicateCharI !n !c
693673 | n < 0 = empty
694674 | otherwise = Stream next 0 (fromIntegral n) -- HINT maybe too low
695675 where
696− next i | i >= n = Done
697− | otherwise = Yield c (i + 1)
676+ next !i | i >= n = Done
677+ | otherwise = Yield c (i + 1)
698678{-# INLINE [0] replicateCharI #-}
699679
700680data RI s = RI !s {-# UNPACK #-} !Int64
@@ -759,24 +739,27 @@ take n0 (Stream next0 s0 len) =
759739 Yield x s' -> Yield x ((n-1) :*: s')
760740{-# INLINE [0] take #-}
761741
742+data Drop a s = NS !s
743+ | JS !a !s
744+
762745-- | /O(n)/ drop n, applied to a stream, returns the suffix of the
763746-- stream after the first @n@ characters, or the empty stream if @n@
764747-- is greater than the length of the stream.
765748drop :: Integral a => a -> Stream Char -> Stream Char
766749drop n0 (Stream next0 s0 len) =
767− Stream next (J n0 :*: s0) (len - fromIntegral (max 0 n0))
750+ Stream next (JS n0 s0) (len - fromIntegral (max 0 n0))
768751 where
769752 {-# INLINE next #-}
770− next (J n :*: s)
771− | n <= 0 = Skip (N :*: s)
753+ next (JS n s)
754+ | n <= 0 = Skip (NS s)
772755 | otherwise = case next0 s of
773756 Done -> Done
774− Skip s' -> Skip (J n :*: s')
775− Yield _ s' -> Skip (J (n-1) :*: s')
776− next (N :*: s) = case next0 s of
757+ Skip s' -> Skip (JS n s')
758+ Yield _ s' -> Skip (JS (n-1) s')
759+ next (NS s) = case next0 s of
777760 Done -> Done
778− Skip s' -> Skip (N :*: s')
779− Yield x s' -> Yield x (N :*: s')
761+ Skip s' -> Skip (NS s')
762+ Yield x s' -> Yield x (NS s')
780763{-# INLINE [0] drop #-}
781764
782765-- | takeWhile, applied to a predicate @p@ and a stream, returns the
@@ -794,18 +777,18 @@ takeWhile p (Stream next0 s0 len) = Stream next s0 len -- HINT maybe too high
794777
795778-- | dropWhile @p @xs returns the suffix remaining after takeWhile @p @xs.
796779dropWhile :: (Char -> Bool) -> Stream Char -> Stream Char
797−dropWhile p (Stream next0 s0 len) = Stream next (S1 :*: s0) len -- HINT maybe too high
780+dropWhile p (Stream next0 s0 len) = Stream next (L s0) len -- HINT maybe too high
798781 where
799782 {-# INLINE next #-}
800− next (S1 :*: s) = case next0 s of
783+ next (L s) = case next0 s of
801784 Done -> Done
802− Skip s' -> Skip (S1 :*: s')
803− Yield x s' | p x -> Skip (S1 :*: s')
804− | otherwise -> Yield x (S2 :*: s')
805− next (S2 :*: s) = case next0 s of
785+ Skip s' -> Skip (L s')
786+ Yield x s' | p x -> Skip (L s')
787+ | otherwise -> Yield x (R s')
788+ next (R s) = case next0 s of
806789 Done -> Done
807− Skip s' -> Skip (S2 :*: s')
808− Yield x s' -> Yield x (S2 :*: s')
790+ Skip s' -> Skip (R s')
791+ Yield x s' -> Yield x (R s')
809792{-# INLINE [0] dropWhile #-}
810793
811794-- | /O(n)/ The 'isPrefixOf' function takes two 'Stream's and returns
@@ -912,21 +895,25 @@ findIndicesI p (Stream next s0 _len) = loop_findIndex 0 s0
912895-------------------------------------------------------------------------------
913896-- * Zipping
914897
898+-- | Strict triple.
899+data Zip a b m = Z1 !a !b
900+ | Z2 !a !b !m
901+
915902-- | zipWith generalises 'zip' by zipping with the function given as
916903-- the first argument, instead of a tupling function.
917904zipWith :: (a -> a -> b) -> Stream a -> Stream a -> Stream b
918905zipWith f (Stream next0 sa0 len1) (Stream next1 sb0 len2) =
919− Stream next (sa0 :*: sb0 :*: N) (smaller len1 len2)
906+ Stream next (Z1 sa0 sb0) (smaller len1 len2)
920907 where
921− next (sa :*: sb :*: N) = case next0 sa of
922− Done -> Done
923− Skip sa' -> Skip (sa' :*: sb :*: N)
924− Yield a sa' -> Skip (sa' :*: sb :*: J a)
925−
926− next (sa' :*: sb :*: J a) = case next1 sb of
927− Done -> Done
928− Skip sb' -> Skip (sa' :*: sb' :*: J a)
929− Yield b sb' -> Yield (f a b) (sa' :*: sb' :*: N)
908+ next (Z1 sa sb) = case next0 sa of
909+ Done -> Done
910+ Skip sa' -> Skip (Z1 sa' sb)
911+ Yield a sa' -> Skip (Z2 sa' sb a)
912+
913+ next (Z2 sa' sb a) = case next1 sb of
914+ Done -> Done
915+ Skip sb' -> Skip (Z2 sa' sb' a)
916+ Yield b sb' -> Yield (f a b) (Z1 sa' sb')
930917{-# INLINE [0] zipWith #-}
931918
932919-- | /O(n)/ The 'countCharI' function returns the number of times the
Data/Text/Internal/Fusion/Types.hs+8−13
@@ -20,13 +20,11 @@
2020module Data.Text.Internal.Fusion.Types
2121 (
2222 CC(..)
23− , M(..)
24− , M8
2523 , PairS(..)
24+ , Scan(..)
2625 , RS(..)
2726 , Step(..)
2827 , Stream(..)
29− , Switch(..)
3028 , empty
3129 ) where
3230
@@ -36,26 +34,23 @@ import Data.Word (Word8)
3634-- | Specialised tuple for case conversion.
3735data CC s = CC !s {-# UNPACK #-} !Char {-# UNPACK #-} !Char
3836
39−-- | Specialised, strict Maybe-like type.
40−data M a = N
41− | J !a
42−
43−type M8 = M Word8
44−
45−-- Restreaming state.
37+-- | Restreaming state.
4638data RS s
4739 = RS0 !s
4840 | RS1 !s {-# UNPACK #-} !Word8
4941 | RS2 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
5042 | RS3 !s {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
5143
52−infixl 2 :*:
44+-- | Strict pair.
5345data PairS a b = !a :*: !b
5446 -- deriving (Eq, Ord, Show)
47+infixl 2 :*:
5548
56−-- | Allow a function over a stream to switch between two states.
57−data Switch = S1 | S2
49+-- | An intermediate result in a scan.
50+data Scan s = Scan1 {-# UNPACK #-} !Char !s
51+ | Scan2 {-# UNPACK #-} !Char !s
5852
53+-- | Intermediate result in a processing pipeline.
5954data Step s a = Done
6055 | Skip !s
6156 | Yield !a !s
Data/Text/Internal/Read.hs+3−3
@@ -7,7 +7,7 @@
77-- Stability : experimental
88-- Portability : GHC
99--
10−-- Common internal functiopns for reading textual data.
10+-- Common internal functions for reading textual data.
1111module Data.Text.Internal.Read
1212 (
1313 IReader
@@ -18,7 +18,7 @@ module Data.Text.Internal.Read
1818 , perhaps
1919 ) where
2020
21−import Control.Applicative (Applicative(..))
21+import Control.Applicative as App (Applicative(..))
2222import Control.Arrow (first)
2323import Control.Monad (ap)
2424import Data.Char (ord)
@@ -38,7 +38,7 @@ instance Applicative (IParser t) where
3838 (<*>) = ap
3939
4040instance Monad (IParser t) where
41− return = pure
41+ return = App.pure
4242 m >>= k = P $ \t -> case runP m t of
4343 Left err -> Left err
4444 Right (a,t') -> runP (k a) t'
Data/Text/Lazy.hs+147−21
@@ -125,7 +125,10 @@ module Data.Text.Lazy
125125 , mapAccumR
126126
127127 -- ** Generation and unfolding
128+ , repeat
128129 , replicate
130+ , cycle
131+ , iterate
129132 , unfoldr
130133 , unfoldrN
131134
@@ -137,6 +140,7 @@ module Data.Text.Lazy
137140 , drop
138141 , dropEnd
139142 , takeWhile
143+ , takeWhileEnd
140144 , dropWhile
141145 , dropWhileEnd
142146 , dropAround
@@ -207,9 +211,13 @@ import Control.DeepSeq (NFData(..))
207211import Data.Int (Int64)
208212import qualified Data.List as L
209213import Data.Char (isSpace)
210−import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf))
211−import Data.Data (mkNoRepType)
214+import Data.Data (Data(gfoldl, toConstr, gunfold, dataTypeOf), constrIndex,
215+ Constr, mkConstr, DataType, mkDataType, Fixity(Prefix))
216+import Data.Binary (Binary(get, put))
212217import Data.Monoid (Monoid(..))
218+#if MIN_VERSION_base(4,9,0)
219+import Data.Semigroup (Semigroup(..))
220+#endif
213221import Data.String (IsString(..))
214222import qualified Data.Text as T
215223import qualified Data.Text.Internal as T
@@ -218,8 +226,10 @@ import qualified Data.Text.Unsafe as T
218226import qualified Data.Text.Internal.Lazy.Fusion as S
219227import Data.Text.Internal.Fusion.Types (PairS(..))
220228import Data.Text.Internal.Lazy.Fusion (stream, unstream)
221−import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks, foldrChunks)
229+import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldlChunks,
230+ foldrChunks, smallChunkSize)
222231import Data.Text.Internal (firstf, safe, text)
232+import Data.Text.Lazy.Encoding (decodeUtf8', encodeUtf8)
223233import qualified Data.Text.Internal.Functions as F
224234import Data.Text.Internal.Lazy.Search (indices)
225235#if __GLASGOW_HASKELL__ >= 702
@@ -231,6 +241,9 @@ import qualified GHC.Base as GHC
231241import qualified GHC.Exts as Exts
232242#endif
233243import GHC.Prim (Addr#)
244+#if MIN_VERSION_base(4,7,0)
245+import Text.Printf (PrintfArg, formatArg, formatString)
246+#endif
234247
235248-- $fusion
236249--
@@ -328,9 +341,21 @@ instance Show Text where
328341instance Read Text where
329342 readsPrec p str = [(pack x,y) | (x,y) <- readsPrec p str]
330343
344+#if MIN_VERSION_base(4,9,0)
345+-- Semigroup orphan instances for older GHCs are provided by
346+-- 'semigroups` package
347+
348+instance Semigroup Text where
349+ (<>) = append
350+#endif
351+
331352instance Monoid Text where
332353 mempty = empty
354+#if MIN_VERSION_base(4,9,0)
355+ mappend = (<>) -- future-proof definition
356+#else
333357 mappend = append
358+#endif
334359 mconcat = concat
335360
336361instance IsString Text where
@@ -349,11 +374,38 @@ instance NFData Text where
349374 rnf (Chunk _ ts) = rnf ts
350375#endif
351376
377+instance Binary Text where
378+ put t = put (encodeUtf8 t)
379+ get = do
380+ bs <- get
381+ case decodeUtf8' bs of
382+ P.Left exn -> P.fail (P.show exn)
383+ P.Right a -> P.return a
384+
385+-- | This instance preserves data abstraction at the cost of inefficiency.
386+-- We omit reflection services for the sake of data abstraction.
387+--
388+-- This instance was created by copying the updated behavior of
389+-- @"Data.Text".@'Data.Text.Text'
352390instance Data Text where
353391 gfoldl f z txt = z pack `f` (unpack txt)
354− toConstr _ = error "Data.Text.Lazy.Text.toConstr"
355− gunfold _ _ = error "Data.Text.Lazy.Text.gunfold"
356− dataTypeOf _ = mkNoRepType "Data.Text.Lazy.Text"
392+ toConstr _ = packConstr
393+ gunfold k z c = case constrIndex c of
394+ 1 -> k (z pack)
395+ _ -> error "Data.Text.Lazy.Text.gunfold"
396+ dataTypeOf _ = textDataType
397+
398+#if MIN_VERSION_base(4,7,0)
399+-- | Only defined for @base-4.7.0.0@ and later
400+instance PrintfArg Text where
401+ formatArg txt = formatString $ unpack txt
402+#endif
403+
404+packConstr :: Constr
405+packConstr = mkConstr textDataType "pack" [] Prefix
406+
407+textDataType :: DataType
408+textDataType = mkDataType "Data.Text.Lazy.Text" [packConstr]
357409
358410-- | /O(n)/ Convert a 'String' into a 'Text'.
359411--
@@ -381,6 +433,14 @@ unpackCString# addr# = unstream (S.streamCString# addr#)
381433 unstream (S.streamList (L.map safe (GHC.unpackCStringUtf8# a)))
382434 = unpackCString# a #-}
383435
436+{-# RULES "LAZY TEXT empty literal"
437+ unstream (S.streamList (L.map safe []))
438+ = Empty #-}
439+
440+{-# RULES "LAZY TEXT empty literal" forall a.
441+ unstream (S.streamList (L.map safe [a]))
442+ = Chunk (T.singleton a) Empty #-}
443+
384444-- | /O(1)/ Convert a character into a Text. Subject to fusion.
385445-- Performs replacement on invalid scalar values.
386446singleton :: Char -> Text
@@ -485,7 +545,7 @@ tail Empty = emptyError "tail"
485545 unstream (S.tail (stream t)) = tail t
486546 #-}
487547
488−-- | /O(1)/ Returns all but the last character of a 'Text', which must
548+-- | /O(n\/c)/ Returns all but the last character of a 'Text', which must
489549-- be non-empty. Subject to fusion.
490550init :: Text -> Text
491551init (Chunk t0 ts0) = go t0 ts0
@@ -521,7 +581,7 @@ isSingleton :: Text -> Bool
521581isSingleton = S.isSingleton . stream
522582{-# INLINE isSingleton #-}
523583
524−-- | /O(1)/ Returns the last character of a 'Text', which must be
584+-- | /O(n\/c)/ Returns the last character of a 'Text', which must be
525585-- non-empty. Subject to fusion.
526586last :: Text -> Char
527587last Empty = emptyError "last"
@@ -655,13 +715,36 @@ reverse = rev Empty
655715 where rev a Empty = a
656716 rev a (Chunk t ts) = rev (Chunk (T.reverse t) a) ts
657717
658−-- | /O(m+n)/ Replace every occurrence of one substring with another.
718+-- | /O(m+n)/ Replace every non-overlapping occurrence of @needle@ in
719+-- @haystack@ with @replacement@.
720+--
721+-- This function behaves as though it was defined as follows:
722+--
723+-- @
724+-- replace needle replacement haystack =
725+-- 'intercalate' replacement ('splitOn' needle haystack)
726+-- @
727+--
728+-- As this suggests, each occurrence is replaced exactly once. So if
729+-- @needle@ occurs in @replacement@, that occurrence will /not/ itself
730+-- be replaced recursively:
731+--
732+-- > replace "oo" "foo" "oo" == "foo"
733+--
734+-- In cases where several instances of @needle@ overlap, only the
735+-- first one will be replaced:
736+--
737+-- > replace "ofo" "bar" "ofofo" == "barfo"
659738--
660739-- In (unlikely) bad cases, this function's time complexity degrades
661740-- towards /O(n*m)/.
662−replace :: Text -- ^ Text to search for
663− -> Text -- ^ Replacement text
664− -> Text -- ^ Input text
741+replace :: Text
742+ -- ^ @needle@ to search for. If this string is empty, an
743+ -- error will occur.
744+ -> Text
745+ -- ^ @replacement@ to replace @needle@ with.
746+ -> Text
747+ -- ^ @haystack@ in which to search.
665748 -> Text
666749replace s d = intercalate d . splitOn s
667750{-# INLINE replace #-}
@@ -798,13 +881,13 @@ concatMap f = concat . foldr ((:) . f) []
798881{-# INLINE concatMap #-}
799882
800883-- | /O(n)/ 'any' @p@ @t@ determines whether any character in the
801−-- 'Text' @t@ satisifes the predicate @p@. Subject to fusion.
884+-- 'Text' @t@ satisfies the predicate @p@. Subject to fusion.
802885any :: (Char -> Bool) -> Text -> Bool
803886any p t = S.any p (stream t)
804887{-# INLINE any #-}
805888
806889-- | /O(n)/ 'all' @p@ @t@ determines whether all characters in the
807−-- 'Text' @t@ satisify the predicate @p@. Subject to fusion.
890+-- 'Text' @t@ satisfy the predicate @p@. Subject to fusion.
808891all :: (Char -> Bool) -> Text -> Bool
809892all p t = S.all p (stream t)
810893{-# INLINE all #-}
@@ -887,6 +970,12 @@ mapAccumR f = go
887970 go z Empty = (z, Empty)
888971{-# INLINE mapAccumR #-}
889972
973+-- | @'repeat' x@ is an infinite 'Text', with @x@ the value of every
974+-- element.
975+repeat :: Char -> Text
976+repeat c = let t = Chunk (T.replicate smallChunkSize (T.singleton c)) t
977+ in t
978+
890979-- | /O(n*m)/ 'replicate' @n@ @t@ is a 'Text' consisting of the input
891980-- @t@ repeated @n@ times.
892981replicate :: Int64 -> Text -> Text
@@ -898,6 +987,21 @@ replicate n t
898987 | otherwise = t : rep (i+1)
899988{-# INLINE [1] replicate #-}
900989
990+-- | 'cycle' ties a finite, non-empty 'Text' into a circular one, or
991+-- equivalently, the infinite repetition of the original 'Text'.
992+cycle :: Text -> Text
993+cycle Empty = emptyError "cycle"
994+cycle t = let t' = foldrChunks Chunk t' t
995+ in t'
996+
997+-- | @'iterate' f x@ returns an infinite 'Text' of repeated applications
998+-- of @f@ to @x@:
999+--
1000+-- > iterate f x == [x, f x, f (f x), ...]
1001+iterate :: (Char -> Char) -> Char -> Text
1002+iterate f c = let t c' = Chunk (T.singleton c') (t (f c'))
1003+ in t c
1004+
9011005-- | /O(n)/ 'replicateChar' @n@ @c@ is a 'Text' of length @n@ with @c@ the
9021006-- value of every element. Subject to fusion.
9031007replicateChar :: Int64 -> Char -> Text
@@ -1040,6 +1144,20 @@ takeWhile p t0 = takeWhile' t0
10401144"LAZY TEXT takeWhile -> unfused" [1] forall p t.
10411145 unstream (S.takeWhile p (stream t)) = takeWhile p t
10421146 #-}
1147+-- | /O(n)/ 'takeWhileEnd', applied to a predicate @p@ and a 'Text',
1148+-- returns the longest suffix (possibly empty) of elements that
1149+-- satisfy @p@.
1150+-- Examples:
1151+--
1152+-- > takeWhileEnd (=='o') "foo" == "oo"
1153+takeWhileEnd :: (Char -> Bool) -> Text -> Text
1154+takeWhileEnd p = takeChunk empty . L.reverse . toChunks
1155+ where takeChunk acc [] = acc
1156+ takeChunk acc (t:ts) = if T.length t' < T.length t
1157+ then (Chunk t' acc)
1158+ else takeChunk (Chunk t' acc) ts
1159+ where t' = T.takeWhileEnd p t
1160+{-# INLINE takeWhileEnd #-}
10431161
10441162-- | /O(n)/ 'dropWhile' @p@ @t@ returns the suffix remaining after
10451163-- 'takeWhile' @p@ @t@. Subject to fusion.
@@ -1058,9 +1176,11 @@ dropWhile p t0 = dropWhile' t0
10581176"LAZY TEXT dropWhile -> unfused" [1] forall p t.
10591177 unstream (S.dropWhile p (stream t)) = dropWhile p t
10601178 #-}
1179+
10611180-- | /O(n)/ 'dropWhileEnd' @p@ @t@ returns the prefix remaining after
1062−-- dropping characters that fail the predicate @p@ from the end of
1181+-- dropping characters that satisfy the predicate @p@ from the end of
10631182-- @t@.
1183+--
10641184-- Examples:
10651185--
10661186-- > dropWhileEnd (=='.') "foo..." == "foo"
@@ -1077,7 +1197,7 @@ dropWhileEnd p = go
10771197{-# INLINE dropWhileEnd #-}
10781198
10791199-- | /O(n)/ 'dropAround' @p@ @t@ returns the substring remaining after
1080−-- dropping characters that fail the predicate @p@ from both the
1200+-- dropping characters that satisfy the predicate @p@ from both the
10811201-- beginning and end of @t@. Subject to fusion.
10821202dropAround :: (Char -> Bool) -> Text -> Text
10831203dropAround p = dropWhile p . dropWhileEnd p
@@ -1272,9 +1392,10 @@ tails ts@(Chunk t ts')
12721392-- copies to create substrings; they just construct new 'Text's that
12731393-- are slices of the original.
12741394
1275−-- | /O(m+n)/ Break a 'Text' into pieces separated by the first
1276−-- 'Text' argument, consuming the delimiter. An empty delimiter is
1277−-- invalid, and will cause an error to be raised.
1395+-- | /O(m+n)/ Break a 'Text' into pieces separated by the first 'Text'
1396+-- argument (which cannot be an empty string), consuming the
1397+-- delimiter. An empty delimiter is invalid, and will cause an error
1398+-- to be raised.
12781399--
12791400-- Examples:
12801401--
@@ -1287,13 +1408,18 @@ tails ts@(Chunk t ts')
12871408-- > intercalate s . splitOn s == id
12881409-- > splitOn (singleton c) == split (==c)
12891410--
1411+-- (Note: the string @s@ to split on above cannot be empty.)
1412+--
12901413-- This function is strict in its first argument, and lazy in its
12911414-- second.
12921415--
12931416-- In (unlikely) bad cases, this function's time complexity degrades
12941417-- towards /O(n*m)/.
1295−splitOn :: Text -- ^ Text to split on
1296− -> Text -- ^ Input text
1418+splitOn :: Text
1419+ -- ^ String to split on. If this string is empty, an error
1420+ -- will occur.
1421+ -> Text
1422+ -- ^ Input text.
12971423 -> [Text]
12981424splitOn pat src
12991425 | null pat = emptyError "splitOn"
Data/Text/Lazy/Builder.hs+1−1
@@ -8,7 +8,7 @@
88-- Module : Data.Text.Lazy.Builder
99-- Copyright : (c) 2013 Bryan O'Sullivan
1010-- (c) 2010 Johan Tibell
11−-- License : BSD3-style (see LICENSE)
11+-- License : BSD-style (see LICENSE)
1212--
1313-- Maintainer : Johan Tibell <johan.tibell@gmail.com>
1414-- Stability : experimental
Data/Text/Lazy/Builder/Int.hs+19−14
@@ -1,4 +1,5 @@
1−{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, UnboxedTuples #-}
1+{-# LANGUAGE BangPatterns, CPP, MagicHash, RankNTypes, ScopedTypeVariables,
2+ UnboxedTuples #-}
23#if __GLASGOW_HASKELL__ >= 702
34{-# LANGUAGE Trustworthy #-}
45#endif
@@ -6,7 +7,7 @@
67-- Module: Data.Text.Lazy.Builder.Int
78-- Copyright: (c) 2013 Bryan O'Sullivan
89-- (c) 2011 MailRank, Inc.
9−-- License: BSD3
10+-- License: BSD-style
1011-- Maintainer: Bryan O'Sullivan <bos@serpentine.com>
1112-- Stability: experimental
1213-- Portability: portable
@@ -34,7 +35,7 @@ import Control.Monad.ST
3435
3536#ifdef __GLASGOW_HASKELL__
3637# if defined(INTEGER_GMP)
37−import GHC.Integer.GMP.Internals
38+import GHC.Integer.GMP.Internals (Integer(S#))
3839# elif defined(INTEGER_SIMPLE)
3940import GHC.Integer
4041# else
@@ -54,7 +55,7 @@ decimal :: Integral a => a -> Builder
5455{-# RULES "decimal/Int16" decimal = boundedDecimal :: Int16 -> Builder #-}
5556{-# RULES "decimal/Int32" decimal = boundedDecimal :: Int32 -> Builder #-}
5657{-# RULES "decimal/Int64" decimal = boundedDecimal :: Int64 -> Builder #-}
57−{-# RULES "decimal/Word" decimal = positive :: Word -> Builder #-}
58+{-# RULES "decimal/Word" decimal = positive :: Data.Word.Word -> Builder #-}
5859{-# RULES "decimal/Word8" decimal = positive :: Word8 -> Builder #-}
5960{-# RULES "decimal/Word16" decimal = positive :: Word16 -> Builder #-}
6061{-# RULES "decimal/Word32" decimal = positive :: Word32 -> Builder #-}
@@ -133,8 +134,15 @@ i2w v = zero + fromIntegral v
133134
134135countDigits :: (Integral a) => a -> Int
135136{-# INLINE countDigits #-}
136−countDigits v0 = go 1 (fromIntegral v0 :: Word64)
137− where go !k v
137+countDigits v0
138+ | fromIntegral v64 == v0 = go 1 v64
139+ | otherwise = goBig 1 (fromIntegral v0)
140+ where v64 = fromIntegral v0
141+ goBig !k (v :: Integer)
142+ | v > big = goBig (k + 19) (v `quot` big)
143+ | otherwise = go k (fromIntegral v)
144+ big = 10000000000000000000
145+ go !k (v :: Word64)
138146 | v < 10 = k
139147 | v < 100 = k + 1
140148 | v < 1000 = k + 2
@@ -186,19 +194,12 @@ hexDigit n
186194 | otherwise = singleton $! toEnum (fromIntegral n + 87)
187195{-# INLINE hexDigit #-}
188196
189−int :: Int -> Builder
190−int = decimal
191−{-# INLINE int #-}
192−
193197data T = T !Integer !Int
194198
195199integer :: Int -> Integer -> Builder
196200#ifdef INTEGER_GMP
197201integer 10 (S# i#) = decimal (I# i#)
198202integer 16 (S# i#) = hexadecimal (I# i#)
199−#else
200−integer 10 i = decimal i
201−integer 16 i = hexadecimal i
202203#endif
203204integer base i
204205 | i < 0 = singleton '-' <> go (-i)
@@ -246,7 +247,11 @@ integer base i
246247 PAIR(x,y) -> pblock q <> pblock r <> putB ns
247248 where q = fromInteger x
248249 r = fromInteger y
249− putB _ = mempty
250+ putB _ = Data.Monoid.mempty
251+
252+ int :: Int -> Builder
253+ int x | base == 10 = decimal x
254+ | otherwise = hexadecimal x
250255
251256 pblock = loop maxDigits
252257 where
Data/Text/Lazy/Encoding.hs+9−21
@@ -46,30 +46,25 @@ module Data.Text.Lazy.Encoding
4646 , encodeUtf32LE
4747 , encodeUtf32BE
4848
49−#if MIN_VERSION_bytestring(0,10,4)
5049 -- * Encoding Text using ByteString Builders
5150 , encodeUtf8Builder
5251 , encodeUtf8BuilderEscaped
53−#endif
5452 ) where
5553
5654import Control.Exception (evaluate, try)
55+import Data.Monoid (Monoid(..))
5756import Data.Text.Encoding.Error (OnDecodeError, UnicodeException, strictDecode)
5857import Data.Text.Internal.Lazy (Text(..), chunk, empty, foldrChunks)
59−import qualified Data.ByteString as S
60−import qualified Data.ByteString.Lazy as B
61−import qualified Data.ByteString.Lazy.Internal as B
62−import qualified Data.ByteString.Unsafe as B
63−#if MIN_VERSION_bytestring(0,10,4)
6458import Data.Word (Word8)
65−import Data.Monoid (mempty, (<>))
59+import qualified Data.ByteString as S
6660import qualified Data.ByteString.Builder as B
6761import qualified Data.ByteString.Builder.Extra as B (safeStrategy, toLazyByteStringWith)
6862import qualified Data.ByteString.Builder.Prim as BP
63+import qualified Data.ByteString.Lazy as B
64+import qualified Data.ByteString.Lazy.Internal as B
65+import qualified Data.ByteString.Unsafe as B
6966import qualified Data.Text as T
70−#endif
7167import qualified Data.Text.Encoding as TE
72−import qualified Data.Text.Lazy as L
7368import qualified Data.Text.Internal.Lazy.Encoding.Fusion as E
7469import qualified Data.Text.Internal.Lazy.Fusion as F
7570import Data.Text.Unsafe (unsafeDupablePerformIO)
@@ -89,8 +84,6 @@ import Data.Text.Unsafe (unsafeDupablePerformIO)
8984
9085-- | /Deprecated/. Decode a 'ByteString' containing 7-bit ASCII
9186-- encoded text.
92−--
93−-- This function is deprecated. Use 'decodeLatin1' instead.
9487decodeASCII :: B.ByteString -> Text
9588decodeASCII = decodeUtf8
9689{-# DEPRECATED decodeASCII "Use decodeUtf8 instead" #-}
@@ -112,7 +105,7 @@ decodeUtf8With onErr (B.Chunk b0 bs0) =
112105 | S.null l = empty
113106 | otherwise = case onErr desc (Just (B.unsafeHead l)) of
114107 Nothing -> empty
115− Just c -> L.singleton c
108+ Just c -> Chunk (T.singleton c) Empty
116109 desc = "Data.Text.Lazy.Encoding.decodeUtf8With: Invalid UTF-8 stream"
117110decodeUtf8With _ _ = empty
118111
@@ -149,7 +142,6 @@ decodeUtf8' bs = unsafeDupablePerformIO $ do
149142{-# INLINE decodeUtf8' #-}
150143
151144encodeUtf8 :: Text -> B.ByteString
152−#if MIN_VERSION_bytestring(0,10,4)
153145encodeUtf8 Empty = B.empty
154146encodeUtf8 lt@(Chunk t _) =
155147 B.toLazyByteStringWith strategy B.empty $ encodeUtf8Builder lt
@@ -163,17 +155,13 @@ encodeUtf8 lt@(Chunk t _) =
163155 strategy = B.safeStrategy firstChunkSize B.defaultChunkSize
164156
165157encodeUtf8Builder :: Text -> B.Builder
166−encodeUtf8Builder = foldrChunks (\c b -> TE.encodeUtf8Builder c <> b) mempty
158+encodeUtf8Builder =
159+ foldrChunks (\c b -> TE.encodeUtf8Builder c `mappend` b) Data.Monoid.mempty
167160
168161{-# INLINE encodeUtf8BuilderEscaped #-}
169162encodeUtf8BuilderEscaped :: BP.BoundedPrim Word8 -> Text -> B.Builder
170163encodeUtf8BuilderEscaped prim =
171− foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c <> b) mempty
172−
173−#else
174−encodeUtf8 (Chunk c cs) = B.Chunk (TE.encodeUtf8 c) (encodeUtf8 cs)
175−encodeUtf8 Empty = B.Empty
176−#endif
164+ foldrChunks (\c b -> TE.encodeUtf8BuilderEscaped prim c `mappend` b) mempty
177165
178166-- | Decode text from little endian UTF-16 encoding.
179167decodeUtf16LEWith :: OnDecodeError -> B.ByteString -> Text
Data/Text/Lazy/Read.hs+4−2
@@ -1,5 +1,7 @@
11{-# LANGUAGE OverloadedStrings, CPP #-}
2−#if __GLASGOW_HASKELL__ >= 702
2+#if __GLASGOW_HASKELL__ >= 704
3+{-# LANGUAGE Safe #-}
4+#elif __GLASGOW_HASKELL__ >= 702
35{-# LANGUAGE Trustworthy #-}
46#endif
57
@@ -54,7 +56,7 @@ decimal :: Integral a => Reader a
5456{-# SPECIALIZE decimal :: Reader Int32 #-}
5557{-# SPECIALIZE decimal :: Reader Int64 #-}
5658{-# SPECIALIZE decimal :: Reader Integer #-}
57−{-# SPECIALIZE decimal :: Reader Word #-}
59+{-# SPECIALIZE decimal :: Reader Data.Word.Word #-}
5860{-# SPECIALIZE decimal :: Reader Word8 #-}
5961{-# SPECIALIZE decimal :: Reader Word16 #-}
6062{-# SPECIALIZE decimal :: Reader Word32 #-}
Data/Text/Read.hs+1−1
@@ -55,7 +55,7 @@ decimal :: Integral a => Reader a
5555{-# SPECIALIZE decimal :: Reader Int32 #-}
5656{-# SPECIALIZE decimal :: Reader Int64 #-}
5757{-# SPECIALIZE decimal :: Reader Integer #-}
58−{-# SPECIALIZE decimal :: Reader Word #-}
58+{-# SPECIALIZE decimal :: Reader Data.Word.Word #-}
5959{-# SPECIALIZE decimal :: Reader Word8 #-}
6060{-# SPECIALIZE decimal :: Reader Word16 #-}
6161{-# SPECIALIZE decimal :: Reader Word32 #-}
Data/Text/Show.hsnew file+89
@@ -0,0 +1,89 @@
1+{-# LANGUAGE CPP, MagicHash #-}
2+{-# OPTIONS_GHC -fno-warn-orphans #-}
3+#if __GLASGOW_HASKELL__ >= 702
4+{-# LANGUAGE Trustworthy #-}
5+#endif
6+
7+-- |
8+-- Module : Data.Text.Show
9+-- Copyright : (c) 2009-2015 Bryan O'Sullivan
10+--
11+-- License : BSD-style
12+-- Maintainer : bos@serpentine.com
13+-- Stability : experimental
14+-- Portability : GHC
15+
16+module Data.Text.Show
17+ (
18+ singleton
19+ , unpack
20+ , unpackCString#
21+ ) where
22+
23+import Control.Monad.ST (ST)
24+import Data.Text.Internal (Text(..), empty_, safe)
25+import Data.Text.Internal.Fusion (stream, unstream)
26+import Data.Text.Internal.Unsafe.Char (unsafeWrite)
27+import GHC.Prim (Addr#)
28+import qualified Data.Text.Array as A
29+import qualified Data.Text.Internal.Fusion.Common as S
30+
31+#if __GLASGOW_HASKELL__ >= 702
32+import qualified GHC.CString as GHC
33+#else
34+import qualified GHC.Base as GHC
35+#endif
36+
37+instance Show Text where
38+ showsPrec p ps r = showsPrec p (unpack ps) r
39+
40+-- | /O(n)/ Convert a 'Text' into a 'String'. Subject to fusion.
41+unpack :: Text -> String
42+unpack = S.unstreamList . stream
43+{-# INLINE [1] unpack #-}
44+
45+-- | /O(n)/ Convert a literal string into a 'Text'. Subject to
46+-- fusion.
47+--
48+-- This is exposed solely for people writing GHC rewrite rules.
49+unpackCString# :: Addr# -> Text
50+unpackCString# addr# = unstream (S.streamCString# addr#)
51+{-# NOINLINE unpackCString# #-}
52+
53+{-# RULES "TEXT literal" [1] forall a.
54+ unstream (S.map safe (S.streamList (GHC.unpackCString# a)))
55+ = unpackCString# a #-}
56+
57+{-# RULES "TEXT literal UTF8" [1] forall a.
58+ unstream (S.map safe (S.streamList (GHC.unpackCStringUtf8# a)))
59+ = unpackCString# a #-}
60+
61+{-# RULES "TEXT empty literal" [1]
62+ unstream (S.map safe (S.streamList []))
63+ = empty_ #-}
64+
65+{-# RULES "TEXT singleton literal" [1] forall a.
66+ unstream (S.map safe (S.streamList [a]))
67+ = singleton_ a #-}
68+
69+-- | /O(1)/ Convert a character into a Text. Subject to fusion.
70+-- Performs replacement on invalid scalar values.
71+singleton :: Char -> Text
72+singleton = unstream . S.singleton . safe
73+{-# INLINE [1] singleton #-}
74+
75+{-# RULES "TEXT singleton" forall a.
76+ unstream (S.singleton (safe a))
77+ = singleton_ a #-}
78+
79+-- This is intended to reduce inlining bloat.
80+singleton_ :: Char -> Text
81+singleton_ c = Text (A.run x) 0 len
82+ where x :: ST s (A.MArray s)
83+ x = do arr <- A.new len
84+ _ <- unsafeWrite arr 0 d
85+ return arr
86+ len | d < '\x10000' = 1
87+ | otherwise = 2
88+ d = safe c
89+{-# NOINLINE singleton_ #-}
benchmarks/haskell/Benchmarks.hs+1−1
@@ -53,7 +53,7 @@ benchmarks = do
5353 , Mul.benchmark
5454 , Pure.benchmark "tiny" (tf "tiny.txt")
5555 , Pure.benchmark "ascii" (tf "ascii-small.txt")
56− , Pure.benchmark "france" (tf "france.html")
56+ -- , Pure.benchmark "france" (tf "france.html")
5757 , Pure.benchmark "russian" (tf "russian-small.txt")
5858 , Pure.benchmark "japanese" (tf "japanese.txt")
5959 , ReadNumbers.benchmark (tf "numbers.txt")
benchmarks/haskell/Benchmarks/DecodeUtf8.hs+2−2
@@ -25,7 +25,7 @@ import Foreign.Ptr (Ptr, plusPtr)
2525import Foreign.ForeignPtr (withForeignPtr)
2626import Data.Word (Word8)
2727import qualified Criterion as C
28−import Criterion (Benchmark, bgroup, nf)
28+import Criterion (Benchmark, bgroup, nf, whnfIO)
2929import qualified Codec.Binary.UTF8.Generic as U8
3030import qualified Data.ByteString as B
3131import qualified Data.ByteString.Lazy as BL
@@ -48,7 +48,7 @@ benchmark kind fp = do
4848 return $ bgroup "DecodeUtf8"
4949 [ bench "Strict" $ nf T.decodeUtf8 bs
5050 , bench "Stream" $ nf decodeStream lbs
51− , bench "IConv" $ iconv bs
51+ , bench "IConv" $ whnfIO $ iconv bs
5252 , bench "StrictLength" $ nf (T.length . T.decodeUtf8) bs
5353 , bench "StrictInitLength" $ nf (T.length . T.init . T.decodeUtf8) bs
5454 , bench "Lazy" $ nf TL.decodeUtf8 lbs
benchmarks/haskell/Benchmarks/FileRead.hs+11−11
@@ -8,8 +8,8 @@ module Benchmarks.FileRead
88 ( benchmark
99 ) where
1010
11−import Control.Exception (evaluate)
12−import Criterion (Benchmark, bgroup, bench)
11+import Control.Applicative ((<$>))
12+import Criterion (Benchmark, bgroup, bench, whnfIO)
1313import qualified Data.ByteString as SB
1414import qualified Data.ByteString.Lazy as LB
1515import qualified Data.Text as T
@@ -21,13 +21,13 @@ import qualified Data.Text.Lazy.IO as LT
2121
2222benchmark :: FilePath -> IO Benchmark
2323benchmark p = return $ bgroup "FileRead"
24− [ bench "String" $ readFile p >>= evaluate . length
25− , bench "ByteString" $ SB.readFile p >>= evaluate . SB.length
26− , bench "LazyByteString" $ LB.readFile p >>= evaluate . LB.length
27− , bench "Text" $ T.readFile p >>= evaluate . T.length
28− , bench "LazyText" $ LT.readFile p >>= evaluate . LT.length
29− , bench "TextByteString" $
30− SB.readFile p >>= evaluate . T.length . T.decodeUtf8
31− , bench "LazyTextByteString" $
32− LB.readFile p >>= evaluate . LT.length . LT.decodeUtf8
24+ [ bench "String" $ whnfIO $ length <$> readFile p
25+ , bench "ByteString" $ whnfIO $ SB.length <$> SB.readFile p
26+ , bench "LazyByteString" $ whnfIO $ LB.length <$> LB.readFile p
27+ , bench "Text" $ whnfIO $ T.length <$> T.readFile p
28+ , bench "LazyText" $ whnfIO $ LT.length <$> LT.readFile p
29+ , bench "TextByteString" $ whnfIO $
30+ (T.length . T.decodeUtf8) <$> SB.readFile p
31+ , bench "LazyTextByteString" $ whnfIO $
32+ (LT.length . LT.decodeUtf8) <$> LB.readFile p
3333 ]
benchmarks/haskell/Benchmarks/FoldLines.hs+2−2
@@ -10,7 +10,7 @@ module Benchmarks.FoldLines
1010 ( benchmark
1111 ) where
1212
13−import Criterion (Benchmark, bgroup, bench)
13+import Criterion (Benchmark, bgroup, bench, whnfIO)
1414import System.IO
1515import qualified Data.ByteString as B
1616import qualified Data.Text as T
@@ -22,7 +22,7 @@ benchmark fp = return $ bgroup "ReadLines"
2222 , bench "ByteString" $ withHandle $ foldLinesB (\n _ -> n + 1) (0 :: Int)
2323 ]
2424 where
25− withHandle f = do
25+ withHandle f = whnfIO $ do
2626 h <- openFile fp ReadMode
2727 hSetBuffering h (BlockBuffering (Just 16384))
2828 x <- f h
benchmarks/haskell/Benchmarks/Programs/BigTable.hs+2−2
@@ -11,7 +11,7 @@ module Benchmarks.Programs.BigTable
1111 ( benchmark
1212 ) where
1313
14−import Criterion (Benchmark, bench)
14+import Criterion (Benchmark, bench, whnfIO)
1515import Data.Monoid (mappend, mconcat)
1616import Data.Text.Lazy.Builder (Builder, fromText, toLazyText)
1717import Data.Text.Lazy.IO (hPutStr)
@@ -19,7 +19,7 @@ import System.IO (Handle)
1919import qualified Data.Text as T
2020
2121benchmark :: Handle -> IO Benchmark
22−benchmark sink = return $ bench "BigTable" $ do
22+benchmark sink = return $ bench "BigTable" $ whnfIO $ do
2323 hPutStr sink "Content-Type: text/html\n\n<table>"
2424 hPutStr sink . toLazyText . makeTable =<< rows
2525 hPutStr sink "</table>"
benchmarks/haskell/Benchmarks/Programs/Cut.hs+2−2
@@ -16,7 +16,7 @@ module Benchmarks.Programs.Cut
1616 ( benchmark
1717 ) where
1818
19−import Criterion (Benchmark, bgroup, bench)
19+import Criterion (Benchmark, bgroup, bench, whnfIO)
2020import System.IO (Handle, hPutStr)
2121import qualified Data.ByteString as B
2222import qualified Data.ByteString.Char8 as BC
@@ -40,7 +40,7 @@ benchmark p sink from to = return $ bgroup "Cut"
4040 , bench' "LazyTextByteString" lazyTextByteString
4141 ]
4242 where
43− bench' n s = bench n (s p sink from to)
43+ bench' n s = bench n $ whnfIO (s p sink from to)
4444
4545string :: FilePath -> Handle -> Int -> Int -> IO ()
4646string fp sink from to = do
benchmarks/haskell/Benchmarks/Programs/Fold.hs+2−2
@@ -21,7 +21,7 @@ import Data.List (foldl')
2121import Data.List (intersperse)
2222import Data.Monoid (mempty, mappend, mconcat)
2323import System.IO (Handle)
24−import Criterion (Benchmark, bench)
24+import Criterion (Benchmark, bench, whnfIO)
2525import qualified Data.Text as T
2626import qualified Data.Text.IO as T
2727import qualified Data.Text.Lazy.Builder as TLB
@@ -30,7 +30,7 @@ import qualified Data.Text.Lazy.IO as TL
3030
3131benchmark :: FilePath -> Handle -> IO Benchmark
3232benchmark i o = return $
33− bench "Fold" $ T.readFile i >>= TL.hPutStr o . fold 80
33+ bench "Fold" $ whnfIO $ T.readFile i >>= TL.hPutStr o . fold 80
3434
3535-- | We represent a paragraph by a word list
3636--
benchmarks/haskell/Benchmarks/Programs/Sort.hs+10−9
@@ -17,7 +17,7 @@ module Benchmarks.Programs.Sort
1717 ( benchmark
1818 ) where
1919
20−import Criterion (Benchmark, bgroup, bench)
20+import Criterion (Benchmark, bgroup, bench, whnfIO)
2121import Data.Monoid (mconcat)
2222import System.IO (Handle, hPutStr)
2323import qualified Data.ByteString as B
@@ -35,16 +35,17 @@ import qualified Data.Text.Lazy.IO as TL
3535
3636benchmark :: FilePath -> Handle -> IO Benchmark
3737benchmark i o = return $ bgroup "Sort"
38− [ bench "String" $ readFile i >>= hPutStr o . string
39− , bench "ByteString" $ B.readFile i >>= B.hPutStr o . byteString
40− , bench "LazyByteString" $ BL.readFile i >>= BL.hPutStr o . lazyByteString
41− , bench "Text" $ T.readFile i >>= T.hPutStr o . text
42− , bench "LazyText" $ TL.readFile i >>= TL.hPutStr o . lazyText
43− , bench "TextByteString" $ B.readFile i >>=
38+ [ bench "String" $ whnfIO $ readFile i >>= hPutStr o . string
39+ , bench "ByteString" $ whnfIO $ B.readFile i >>= B.hPutStr o . byteString
40+ , bench "LazyByteString" $ whnfIO $
41+ BL.readFile i >>= BL.hPutStr o . lazyByteString
42+ , bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text
43+ , bench "LazyText" $ whnfIO $ TL.readFile i >>= TL.hPutStr o . lazyText
44+ , bench "TextByteString" $ whnfIO $ B.readFile i >>=
4445 B.hPutStr o . T.encodeUtf8 . text . T.decodeUtf8
45− , bench "LazyTextByteString" $ BL.readFile i >>=
46+ , bench "LazyTextByteString" $ whnfIO $ BL.readFile i >>=
4647 BL.hPutStr o . TL.encodeUtf8 . lazyText . TL.decodeUtf8
47− , bench "TextBuilder" $ B.readFile i >>=
48+ , bench "TextBuilder" $ whnfIO $ B.readFile i >>=
4849 BL.hPutStr o . TL.encodeUtf8 . textBuilder . T.decodeUtf8
4950 ]
5051
benchmarks/haskell/Benchmarks/Programs/StripTags.hs+5−5
@@ -15,7 +15,7 @@ module Benchmarks.Programs.StripTags
1515 ( benchmark
1616 ) where
1717
18−import Criterion (Benchmark, bgroup, bench)
18+import Criterion (Benchmark, bgroup, bench, whnfIO)
1919import Data.List (mapAccumL)
2020import System.IO (Handle, hPutStr)
2121import qualified Data.ByteString as B
@@ -26,10 +26,10 @@ import qualified Data.Text.IO as T
2626
2727benchmark :: FilePath -> Handle -> IO Benchmark
2828benchmark i o = return $ bgroup "StripTags"
29− [ bench "String" $ readFile i >>= hPutStr o . string
30− , bench "ByteString" $ B.readFile i >>= B.hPutStr o . byteString
31− , bench "Text" $ T.readFile i >>= T.hPutStr o . text
32− , bench "TextByteString" $
29+ [ bench "String" $ whnfIO $ readFile i >>= hPutStr o . string
30+ , bench "ByteString" $ whnfIO $ B.readFile i >>= B.hPutStr o . byteString
31+ , bench "Text" $ whnfIO $ T.readFile i >>= T.hPutStr o . text
32+ , bench "TextByteString" $ whnfIO $
3333 B.readFile i >>= B.hPutStr o . T.encodeUtf8 . text . T.decodeUtf8
3434 ]
3535
benchmarks/haskell/Benchmarks/Programs/Throughput.hs+8−8
@@ -18,7 +18,7 @@ module Benchmarks.Programs.Throughput
1818 ( benchmark
1919 ) where
2020
21−import Criterion (Benchmark, bgroup, bench)
21+import Criterion (Benchmark, bgroup, bench, whnfIO)
2222import System.IO (Handle, hPutStr)
2323import qualified Data.ByteString as B
2424import qualified Data.ByteString.Lazy as BL
@@ -29,13 +29,13 @@ import qualified Data.Text.Lazy.IO as TL
2929
3030benchmark :: FilePath -> Handle -> IO Benchmark
3131benchmark fp sink = return $ bgroup "Throughput"
32− [ bench "String" $ readFile fp >>= hPutStr sink
33− , bench "ByteString" $ B.readFile fp >>= B.hPutStr sink
34− , bench "LazyByteString" $ BL.readFile fp >>= BL.hPutStr sink
35− , bench "Text" $ T.readFile fp >>= T.hPutStr sink
36− , bench "LazyText" $ TL.readFile fp >>= TL.hPutStr sink
37− , bench "TextByteString" $
32+ [ bench "String" $ whnfIO $ readFile fp >>= hPutStr sink
33+ , bench "ByteString" $ whnfIO $ B.readFile fp >>= B.hPutStr sink
34+ , bench "LazyByteString" $ whnfIO $ BL.readFile fp >>= BL.hPutStr sink
35+ , bench "Text" $ whnfIO $ T.readFile fp >>= T.hPutStr sink
36+ , bench "LazyText" $ whnfIO $ TL.readFile fp >>= TL.hPutStr sink
37+ , bench "TextByteString" $ whnfIO $
3838 B.readFile fp >>= B.hPutStr sink . T.encodeUtf8 . T.decodeUtf8
39− , bench "LazyTextByteString" $
39+ , bench "LazyTextByteString" $ whnfIO $
4040 BL.readFile fp >>= BL.hPutStr sink . TL.encodeUtf8 . TL.decodeUtf8
4141 ]
benchmarks/haskell/Benchmarks/Pure.hs+7
@@ -262,6 +262,13 @@ benchmark kind fp = do
262262 , benchBSL $ nf (BL.map toUpper) bla
263263 , benchS $ nf (L.map toUpper) sa
264264 ]
265+ , bgroup "uncons"
266+ [ benchT $ nf T.uncons ta
267+ , benchTL $ nf TL.uncons tla
268+ , benchBS $ nf BS.uncons bsa
269+ , benchBSL $ nf BL.uncons bla
270+ , benchS $ nf L.uncons sa
271+ ]
265272 , bgroup "words"
266273 [ benchT $ nf T.words ta
267274 , benchTL $ nf TL.words tla
benchmarks/haskell/Benchmarks/ReadNumbers.hs+2−5
@@ -25,8 +25,7 @@ import Data.List (foldl')
2525import Numeric (readDec, readFloat, readHex)
2626import qualified Data.ByteString.Char8 as B
2727import qualified Data.ByteString.Lazy.Char8 as BL
28−import qualified Data.ByteString.Lex.Double as B
29−import qualified Data.ByteString.Lex.Lazy.Double as BL
28+import qualified Data.ByteString.Lex.Fractional as B
3029import qualified Data.Text as T
3130import qualified Data.Text.IO as T
3231import qualified Data.Text.Lazy as TL
@@ -63,12 +62,10 @@ benchmark fp = do
6362 whnf (double . text TL.rational) tl
6463
6564 , bench "DecimalByteString" $ whnf (int . byteString B.readInt) b
66− , bench "DoubleByteString" $ whnf (double . byteString B.readDouble) b
65+ , bench "DoubleByteString" $ whnf (double . byteString B.readDecimal) b
6766
6867 , bench "DecimalLazyByteString" $
6968 whnf (int . byteString BL.readInt) bl
70− , bench "DoubleLazyByteString" $
71− whnf (double . byteString BL.readDouble) bl
7269 ]
7370 where
7471 -- Used for fixing types
benchmarks/haskell/Benchmarks/Stream.hs+16−5
@@ -18,11 +18,11 @@ import qualified Data.Text.Encoding as T
1818import qualified Data.Text.Encoding.Error as E
1919import qualified Data.Text.Internal.Encoding.Fusion as T
2020import qualified Data.Text.Internal.Encoding.Fusion.Common as F
21−import qualified Data.Text.Internal.Fusion as T
21+import qualified Data.Text.Internal.Fusion as F
2222import qualified Data.Text.IO as T
2323import qualified Data.Text.Lazy.Encoding as TL
2424import qualified Data.Text.Internal.Lazy.Encoding.Fusion as TL
25−import qualified Data.Text.Internal.Lazy.Fusion as TL
25+import qualified Data.Text.Internal.Lazy.Fusion as FL
2626import qualified Data.Text.Lazy.IO as TL
2727
2828instance NFData a => NFData (Stream a) where
@@ -53,14 +53,25 @@ benchmark fp = do
5353 !utf32beL = TL.encodeUtf32BE tl
5454
5555 -- For the functions which operate on streams
56− let !s = T.stream t
56+ let !s = F.stream t
5757
5858 return $ bgroup "Stream"
5959
6060 -- Fusion
6161 [ bgroup "stream" $
62− [ bench "Text" $ nf T.stream t
63− , bench "LazyText" $ nf TL.stream tl
62+ [ bench "Text" $ nf F.stream t
63+ , bench "LazyText" $ nf FL.stream tl
64+ ]
65+ -- must perform exactly the same as stream above due to
66+ -- stream/unstream (i.e. stream after unstream) fusion
67+ , bgroup "stream-fusion" $
68+ [ bench "Text" $ nf (F.stream . F.unstream . F.stream) t
69+ , bench "LazyText" $ nf (FL.stream . FL.unstream . FL.stream) tl
70+ ]
71+ -- measure the overhead of unstream after stream
72+ , bgroup "stream-unstream" $
73+ [ bench "Text" $ nf (F.unstream . F.stream) t
74+ , bench "LazyText" $ nf (FL.unstream . FL.stream) tl
6475 ]
6576
6677 -- Encoding.Fusion
benchmarks/text-benchmarks.cabal+14−4
@@ -3,7 +3,7 @@ version: 0.0.0.0
33synopsis: Benchmarks for the text package
44description: Benchmarks for the text package
55homepage: https://bitbucket.org/bos/text
6−license: BSD3
6+license: BSD2
77license-file: ../LICENSE
88author: Jasper Van der Jeugt <jaspervdj@gmail.com>,
99 Bryan O'Sullivan <bos@serpentine.com>,
@@ -15,6 +15,11 @@ build-type: Simple
1515
1616cabal-version: >=1.2
1717
18+flag bytestring-builder
19+ description: Depend on the bytestring-builder package for backwards compatibility.
20+ default: False
21+ manual: False
22+
1823flag llvm
1924 description: use LLVM
2025 default: False
@@ -33,10 +38,9 @@ executable text-benchmarks
3338 build-depends: base == 4.*,
3439 binary,
3540 blaze-builder,
36− bytestring,
37− bytestring-lexing,
41+ bytestring-lexing >= 0.5.0,
3842 containers,
39− criterion >= 0.6.0.1,
43+ criterion >= 0.10.0.0,
4044 deepseq,
4145 directory,
4246 filepath,
@@ -46,6 +50,12 @@ executable text-benchmarks
4650 utf8-string,
4751 vector
4852
53+ if flag(bytestring-builder)
54+ build-depends: bytestring >= 0.9 && < 0.10.4,
55+ bytestring-builder >= 0.10.4
56+ else
57+ build-depends: bytestring >= 0.10.4
58+
4959executable text-multilang
5060 hs-source-dirs: haskell
5161 main-is: Multilang.hs
changelog.md+109
@@ -1,3 +1,112 @@
1+1.2.2.1
2+
3+* The switch to `integer-pure` in 1.2.2.0 was apparently mistaken.
4+ The build flag has been renamed accordingly. Your army of diligent
5+ maintainers apologizes for the churn.
6+
7+* Spec compliance: toCaseFold now follows the Unicode 8.0 spec
8+ (updated from 7.0)
9+
10+* An STG lint error has been fixed
11+
12+1.2.2.0
13+
14+* The `integer-simple` package, upon which this package optionally
15+ depended, has been replaced with `integer-pure`. The build flag has
16+ been renamed accordingly.
17+
18+* Bug fix: For the `Binary` instance, If UTF-8 decoding fails during a
19+ `get`, the error is propagated via `fail` instead of an uncatchable
20+ crash.
21+
22+* New function: `takeWhileEnd`
23+
24+* New instances for the `Text` types:
25+ * if `base` >= 4.7: `PrintfArg`
26+ * if `base` >= 4.9: `Semigroup`
27+
28+1.2.1.3
29+
30+* Bug fix: As it turns out, moving the literal rewrite rules to simplifier
31+ phase 2 does not prevent competition with the `unpack` rule, which is
32+ also active in this phase. Unfortunately this was hidden due to a silly
33+ test environment mistake. Moving literal rules back to phase 1 finally
34+ fixes GHC Trac #10528 correctly.
35+
36+1.2.1.2
37+
38+* Bug fix: Run literal rewrite rules in simplifier phase 2.
39+ The behavior of the simplifier changed in GHC 7.10.2,
40+ causing these rules to fail to fire, leading to poor code generation
41+ and long compilation times. See
42+ [GHC Trac #10528](https://ghc.haskell.org/trac/ghc/ticket/10528).
43+
44+1.2.1.1
45+
46+* Expose unpackCString#, which you should never use.
47+
48+1.2.1.0
49+
50+* Added Binary instances for both Text types. (If you have previously
51+ been using the text-binary package to get a Binary instance, it is
52+ now obsolete.)
53+
54+1.2.0.6
55+
56+* Fixed a space leak in UTF-8 decoding
57+
58+1.2.0.5
59+
60+* Feature parity: repeat, cycle, iterate are now implemented for lazy
61+ Text, and the Data instance is more complete
62+
63+* Build speed: an inliner space explosion has been fixed with toCaseFold
64+
65+* Bug fix: encoding Int to a Builder would infinite-loop if the
66+ integer-simple package was used
67+
68+* Deprecation: OnEncodeError and EncodeError are deprecated, as they
69+ are never used
70+
71+* Internals: some types that are used internally in fusion-related
72+ functions have moved around, been renamed, or been deleted (we don't
73+ bump the major version if .Internal modules change)
74+
75+* Spec compliance: toCaseFold now follows the Unicode 7.0 spec
76+ (updated from 6.3)
77+
78+1.2.0.4
79+
80+* Fixed an incompatibility with base < 4.5
81+
82+1.2.0.3
83+
84+* Update formatRealFloat to correspond to the definition in versions
85+ of base newer than 4.5 (https://github.com/bos/text/issues/105)
86+
87+1.2.0.2
88+
89+* Bumped lower bound on deepseq to 1.4 for compatibility with the
90+ upcoming GHC 7.10
91+
92+1.2.0.1
93+
94+* Fixed a buffer overflow in rendering of large Integers
95+ (https://github.com/bos/text/issues/99)
96+
97+1.2.0.0
98+
99+* Fixed an integer overflow in the replace function
100+ (https://github.com/bos/text/issues/81)
101+
102+* Fixed a hang in lazy decodeUtf8With
103+ (https://github.com/bos/text/issues/87)
104+
105+* Reduced codegen bloat caused by use of empty and single-character
106+ literals
107+
108+* Added an instance of IsList for GHC 7.8 and above
109+
11101.1.1.0
2111
3112* The Data.Data instance now allows gunfold to work, via a virtual
scripts/CaseFolding.hs+1−1
@@ -37,7 +37,7 @@ mapCF :: CaseFolding -> [String]
3737mapCF (CF _ ms) = typ ++ (map nice . filter p $ ms) ++ [last]
3838 where
3939 typ = ["foldMapping :: forall s. Char -> s -> Step (CC s) Char"
40− ,"{-# INLINE foldMapping #-}"]
40+ ,"{-# NOINLINE foldMapping #-}"]
4141 last = "foldMapping c s = Yield (toLower c) (CC s '\\0' '\\0')"
4242 nice c = "-- " ++ name c ++ "\n" ++
4343 "foldMapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (CC s " ++ y ++ " " ++ z ++ ")"
scripts/CaseMapping.hs+3−3
@@ -8,7 +8,7 @@ import SpecialCasing
88main = do
99 args <- getArgs
1010 let oname = case args of
11− [] -> "../Data/Text/Fusion/CaseMapping.hs"
11+ [] -> "../Data/Text/Internal/Fusion/CaseMapping.hs"
1212 [o] -> o
1313 psc <- parseSC "SpecialCasing.txt"
1414 pcf <- parseCF "CaseFolding.txt"
@@ -24,10 +24,10 @@ main = do
2424 mapM_ (hPutStrLn h) $
2525 ["{-# LANGUAGE Rank2Types #-}"
2626 ,"-- AUTOMATICALLY GENERATED - DO NOT EDIT"
27− ,"-- Generated by scripts/SpecialCasing.hs"] ++
27+ ,"-- Generated by scripts/CaseMapping.hs"] ++
2828 comments ++
2929 [""
30− ,"module Data.Text.Fusion.CaseMapping where"
30+ ,"module Data.Text.Internal.Fusion.CaseMapping where"
3131 ,"import Data.Char"
3232 ,"import Data.Text.Internal.Fusion.Types"
3333 ,""]
scripts/SpecialCasing.hs+1−1
@@ -43,7 +43,7 @@ mapSC which access twiddle (SC _ ms) =
4343 typ ++ (map nice . filter p $ ms) ++ [last]
4444 where
4545 typ = [which ++ "Mapping :: forall s. Char -> s -> Step (CC s) Char"
46− ,"{-# INLINE " ++ which ++ "Mapping #-}"]
46+ ,"{-# NOINLINE " ++ which ++ "Mapping #-}"]
4747 last = which ++ "Mapping c s = Yield (to" ++ ucFirst which ++ " c) (CC s '\\0' '\\0')"
4848 nice c = "-- " ++ name c ++ "\n" ++
4949 which ++ "Mapping " ++ showC (code c) ++ " s = Yield " ++ x ++ " (CC s " ++ y ++ " " ++ z ++ ")"
tests/LiteralRuleTest.hsnew file+21
@@ -0,0 +1,21 @@
1+{-# LANGUAGE OverloadedStrings #-}
2+
3+module LiteralRuleTest where
4+
5+import Data.Text (Text)
6+
7+-- This should produce 8 firings of the "TEXT literal" rule
8+strings :: [Text]
9+strings = [ "abstime", "aclitem", "bit", "bool", "box", "bpchar", "bytea", "char" ]
10+
11+-- This should produce 7 firings of the "TEXT literal UTF8" rule
12+utf8Strings :: [Text]
13+utf8Strings = [ "\0abstime", "\0aclitem", "\xfefe bit", "\0bool", "\0box", "\0bpchar", "\0bytea" ]
14+
15+-- This should produce 4 firings of the "TEXT empty literal" rule
16+empties :: [Text]
17+empties = [ "", "", "", "" ]
18+
19+-- This should produce 5 firings of the "TEXT empty literal" rule
20+--singletons :: [Text]
21+--singletons = [ "a", "b", "c", "d", "e" ]
tests/Makefile+6−1
@@ -1,5 +1,10 @@
11count = 1000
22
3+all: coverage literal-rule-test
4+
5+literal-rule-test:
6+ ./literal-rule-test.sh
7+
38coverage: build coverage/hpc_index.html
49
510build: text-test-data
@@ -32,4 +37,4 @@ coverage/hpc_index.html: coverage/coverage.tix
3237clean:
3338 rm -rf dist coverage .hpc
3439
35−.PHONY: build coverage
40+.PHONY: build coverage all literal-rule-test
tests/Tests/Properties.hs+57−9
@@ -1,9 +1,8 @@
1−-- | General quicktest properties for the text library
2−--
1+-- | QuickCheck properties for the text library.
2+
33{-# LANGUAGE BangPatterns, FlexibleInstances, OverloadedStrings,
44 ScopedTypeVariables, TypeSynonymInstances #-}
5−{-# OPTIONS_GHC -fno-enable-rewrite-rules #-}
6−{-# OPTIONS_GHC -fno-warn-missing-signatures #-}
5+{-# OPTIONS_GHC -fno-enable-rewrite-rules -fno-warn-missing-signatures #-}
76module Tests.Properties
87 (
98 tests
@@ -15,7 +14,7 @@ import Data.Bits ((.&.))
1514import Data.Char (chr, isDigit, isHexDigit, isLower, isSpace, isUpper, ord)
1615import Data.Int (Int8, Int16, Int32, Int64)
1716import Data.Monoid (Monoid(..))
18−import Data.String (fromString)
17+import Data.String (IsString(fromString))
1918import Data.Text.Encoding.Error
2019import Data.Text.Foreign
2120import Data.Text.Internal.Encoding.Utf8
@@ -24,7 +23,7 @@ import Data.Text.Internal.Search (indices)
2423import Data.Text.Lazy.Read as TL
2524import Data.Text.Read as T
2625import Data.Word (Word, Word8, Word16, Word32, Word64)
27−import Numeric (showHex)
26+import Numeric (showEFloat, showFFloat, showGFloat, showHex)
2827import Prelude hiding (replicate)
2928import Test.Framework (Test, testGroup)
3029import Test.Framework.Providers.QuickCheck2 (testProperty)
@@ -441,6 +440,10 @@ t_mapAccumR f z = L.mapAccumR f z `eqP` (second unpackS . T.mapAccumR f z)
441440tl_mapAccumR f z = L.mapAccumR f z `eqP` (second unpackS . TL.mapAccumR f z)
442441 where _types = f :: Int -> Char -> (Int,Char)
443442
443+tl_repeat n = (L.take m . L.repeat) `eq`
444+ (unpackS . TL.take (fromIntegral m) . TL.repeat)
445+ where m = fromIntegral (n :: Word8)
446+
444447replicate n l = concat (L.replicate n l)
445448
446449s_replicate n = replicate m `eq`
@@ -452,6 +455,14 @@ tl_replicate n = replicate m `eq`
452455 (unpackS . TL.replicate (fromIntegral m) . packS)
453456 where m = fromIntegral (n :: Word8)
454457
458+tl_cycle n = (L.take m . L.cycle) `eq`
459+ (unpackS . TL.take (fromIntegral m) . TL.cycle . packS)
460+ where m = fromIntegral (n :: Word8)
461+
462+tl_iterate f n = (L.take m . L.iterate f) `eq`
463+ (unpackS . TL.take (fromIntegral m) . TL.iterate f)
464+ where m = fromIntegral (n :: Word8)
465+
455466unf :: Int -> Char -> Maybe (Char, Char)
456467unf n c | fromEnum c * 100 > n = Nothing
457468 | otherwise = Just (c, succ c)
@@ -505,6 +516,10 @@ sf_takeWhile q p = (L.takeWhile p . L.filter q) `eqP`
505516 (unpackS . S.takeWhile p . S.filter q)
506517t_takeWhile p = L.takeWhile p `eqP` (unpackS . T.takeWhile p)
507518tl_takeWhile p = L.takeWhile p `eqP` (unpackS . TL.takeWhile p)
519+t_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`
520+ (unpackS . T.takeWhileEnd p)
521+tl_takeWhileEnd p = (L.reverse . L.takeWhile p . L.reverse) `eqP`
522+ (unpackS . TL.takeWhileEnd p)
508523s_dropWhile p = L.dropWhile p `eqP` (unpackS . S.dropWhile p)
509524s_dropWhile_s p = L.dropWhile p `eqP` (unpackS . S.unstream . S.dropWhile p)
510525sf_dropWhile q p = (L.dropWhile p . L.filter q) `eqP`
@@ -741,6 +756,7 @@ tb_decimal :: (Integral a, Show a) => a -> Bool
741756tb_decimal = (TB.toLazyText . TB.decimal) `eq` (TL.pack . show)
742757
743758tb_decimal_integer (a::Integer) = tb_decimal a
759+tb_decimal_integer_big (Big a) = tb_decimal a
744760tb_decimal_int (a::Int) = tb_decimal a
745761tb_decimal_int8 (a::Int8) = tb_decimal a
746762tb_decimal_int16 (a::Int16) = tb_decimal a
@@ -752,6 +768,11 @@ tb_decimal_word16 (a::Word16) = tb_decimal a
752768tb_decimal_word32 (a::Word32) = tb_decimal a
753769tb_decimal_word64 (a::Word64) = tb_decimal a
754770
771+tb_decimal_big_int (BigBounded (a::Int)) = tb_decimal a
772+tb_decimal_big_int64 (BigBounded (a::Int64)) = tb_decimal a
773+tb_decimal_big_word (BigBounded (a::Word)) = tb_decimal a
774+tb_decimal_big_word64 (BigBounded (a::Word64)) = tb_decimal a
775+
755776tb_hex :: (Integral a, Show a) => a -> Bool
756777tb_hex = (TB.toLazyText . TB.hexadecimal) `eq` (TL.pack . flip showHex "")
757778
@@ -773,6 +794,21 @@ tb_realfloat = (TB.toLazyText . TB.realFloat) `eq` (TL.pack . show)
773794tb_realfloat_float (a::Float) = tb_realfloat a
774795tb_realfloat_double (a::Double) = tb_realfloat a
775796
797+showFloat :: (RealFloat a) => TB.FPFormat -> Maybe Int -> a -> ShowS
798+showFloat TB.Exponent = showEFloat
799+showFloat TB.Fixed = showFFloat
800+showFloat TB.Generic = showGFloat
801+
802+tb_formatRealFloat :: (RealFloat a, Show a) =>
803+ a -> TB.FPFormat -> Precision a -> Property
804+tb_formatRealFloat a fmt prec =
805+ TB.formatRealFloat fmt p a ===
806+ TB.fromString (showFloat fmt p a "")
807+ where p = precision a prec
808+
809+tb_formatRealFloat_float (a::Float) = tb_formatRealFloat a
810+tb_formatRealFloat_double (a::Double) = tb_formatRealFloat a
811+
776812-- Reading.
777813
778814t_decimal (n::Int) s =
@@ -792,7 +828,7 @@ tl_hexadecimal m s ox =
792828 p = if ox then "0x" else ""
793829 n = getPositive m :: Int
794830
795−isFloaty c = c `elem` "+-.0123456789eE"
831+isFloaty c = c `elem` ("+-.0123456789eE" :: String)
796832
797833t_read_rational p tol (n::Double) s =
798834 case p (T.pack (show n) `T.append` t) of
@@ -1074,9 +1110,12 @@ tests =
10741110 ],
10751111
10761112 testGroup "unfolds" [
1113+ testProperty "tl_repeat" tl_repeat,
10771114 testProperty "s_replicate" s_replicate,
10781115 testProperty "t_replicate" t_replicate,
10791116 testProperty "tl_replicate" tl_replicate,
1117+ testProperty "tl_cycle" tl_cycle,
1118+ testProperty "tl_iterate" tl_iterate,
10801119 testProperty "t_unfoldr" t_unfoldr,
10811120 testProperty "tl_unfoldr" tl_unfoldr,
10821121 testProperty "t_unfoldrN" t_unfoldrN,
@@ -1107,6 +1146,8 @@ tests =
11071146 testProperty "sf_takeWhile" sf_takeWhile,
11081147 testProperty "t_takeWhile" t_takeWhile,
11091148 testProperty "tl_takeWhile" tl_takeWhile,
1149+ testProperty "t_takeWhileEnd" t_takeWhileEnd,
1150+ testProperty "tl_takeWhileEnd" tl_takeWhileEnd,
11101151 testProperty "sf_dropWhile" sf_dropWhile,
11111152 testProperty "s_dropWhile" s_dropWhile,
11121153 testProperty "s_dropWhile_s" s_dropWhile_s,
@@ -1252,11 +1293,16 @@ tests =
12521293 testProperty "tb_decimal_int32" tb_decimal_int32,
12531294 testProperty "tb_decimal_int64" tb_decimal_int64,
12541295 testProperty "tb_decimal_integer" tb_decimal_integer,
1296+ testProperty "tb_decimal_integer_big" tb_decimal_integer_big,
12551297 testProperty "tb_decimal_word" tb_decimal_word,
12561298 testProperty "tb_decimal_word8" tb_decimal_word8,
12571299 testProperty "tb_decimal_word16" tb_decimal_word16,
12581300 testProperty "tb_decimal_word32" tb_decimal_word32,
1259− testProperty "tb_decimal_word64" tb_decimal_word64
1301+ testProperty "tb_decimal_word64" tb_decimal_word64,
1302+ testProperty "tb_decimal_big_int" tb_decimal_big_int,
1303+ testProperty "tb_decimal_big_word" tb_decimal_big_word,
1304+ testProperty "tb_decimal_big_int64" tb_decimal_big_int64,
1305+ testProperty "tb_decimal_big_word64" tb_decimal_big_word64
12601306 ],
12611307 testGroup "hexadecimal" [
12621308 testProperty "tb_hexadecimal_int" tb_hexadecimal_int,
@@ -1273,7 +1319,9 @@ tests =
12731319 ],
12741320 testGroup "realfloat" [
12751321 testProperty "tb_realfloat_double" tb_realfloat_double,
1276− testProperty "tb_realfloat_float" tb_realfloat_float
1322+ testProperty "tb_realfloat_float" tb_realfloat_float,
1323+ testProperty "tb_formatRealFloat_float" tb_formatRealFloat_float,
1324+ testProperty "tb_formatRealFloat_double" tb_formatRealFloat_double
12771325 ],
12781326 testProperty "tb_fromText" tb_fromText,
12791327 testProperty "tb_singleton" tb_singleton
tests/Tests/QuickCheckUtils.hs+73−82
@@ -7,48 +7,46 @@
77module Tests.QuickCheckUtils
88 (
99 genUnicode
10− , genUnicodeWith
11− , ascii
12− , plane0
13− , plane1
14− , plane2
15− , plane14
16− , planes
17−
1810 , unsquare
1911 , smallArbitrary
2012
21− , NotEmpty (..)
13+ , BigBounded(..)
14+ , BigInt(..)
15+ , NotEmpty(..)
2216
23− , Small (..)
17+ , Small(..)
2418 , small
2519
20+ , Precision(..)
21+ , precision
22+
2623 , integralRandomR
2724
28− , DecodeErr (..)
25+ , DecodeErr(..)
2926 , genDecodeErr
3027
31− , Stringy (..)
28+ , Stringy(..)
3229 , eq
3330 , eqP
3431
35− , Encoding (..)
32+ , Encoding(..)
3633
3734 , write_read
3835 ) where
3936
37+import Control.Applicative ((<$>))
4038import Control.Arrow (first, (***))
4139import Control.DeepSeq (NFData (..), deepseq)
4240import Control.Exception (bracket)
43−import Data.Bits ((.&.))
44−import Data.Char (chr)
4541import Data.String (IsString, fromString)
4642import Data.Text.Foreign (I16)
43+import Data.Text.Lazy.Builder.RealFloat (FPFormat(..))
4744import Data.Word (Word8, Word16)
4845import Debug.Trace (trace)
49−import System.Random (Random (..), RandomGen)
50−import Test.QuickCheck hiding (Small (..), (.&.))
46+import System.Random (Random(..), RandomGen)
47+import Test.QuickCheck hiding (Fixed(..), Small (..), (.&.))
5148import Test.QuickCheck.Monadic (assert, monadicIO, run)
49+import Test.QuickCheck.Unicode (string)
5250import Tests.Utils
5351import qualified Data.ByteString as B
5452import qualified Data.Text as T
@@ -60,6 +58,14 @@ import qualified Data.Text.Internal.Lazy.Fusion as TLF
6058import qualified Data.Text.Lazy as TL
6159import qualified System.IO as IO
6260
61+#if !MIN_VERSION_base(4,4,0)
62+import Data.Int (Int64)
63+import Data.Word (Word, Word64)
64+#endif
65+
66+genUnicode :: IsString a => Gen a
67+genUnicode = fromString <$> string
68+
6369instance Random I16 where
6470 randomR = integralRandomR
6571 random = randomR (minBound,maxBound)
@@ -73,75 +79,22 @@ instance Arbitrary B.ByteString where
7379 shrink = map B.pack . shrink . B.unpack
7480
7581#if !MIN_VERSION_base(4,4,0)
82+instance Random Int64 where
83+ randomR = integralRandomR
84+ random = randomR (minBound,maxBound)
85+
86+instance Random Word where
87+ randomR = integralRandomR
88+ random = randomR (minBound,maxBound)
89+
7690instance Random Word8 where
7791 randomR = integralRandomR
7892 random = randomR (minBound,maxBound)
79−#endif
8093
81−genUnicode :: IsString a => Gen a
82−genUnicode = genUnicodeWith planes
83−
84−genUnicodeWith :: IsString a => [Gen Int] -> Gen a
85−genUnicodeWith gens = fmap fromString string
86− where
87− string = sized $ \n ->
88− do k <- choose (0,n)
89− sequence [ char | _ <- [1..k] ]
90−
91− excluding :: [a -> Bool] -> Gen a -> Gen a
92− excluding bad gen = loop
93− where
94− loop = do
95− x <- gen
96− if or (map ($ x) bad)
97− then loop
98− else return x
99−
100− reserved = [lowSurrogate, highSurrogate, noncharacter]
101− lowSurrogate c = c >= 0xDC00 && c <= 0xDFFF
102− highSurrogate c = c >= 0xD800 && c <= 0xDBFF
103− noncharacter c = masked == 0xFFFE || masked == 0xFFFF
104− where
105− masked = c .&. 0xFFFF
106−
107− char = chr `fmap` excluding reserved (oneof gens)
108−
109−ascii :: Gen Int
110−ascii = choose (0,0x7F)
111−
112−plane0 :: Gen Int
113−plane0 = choose (0xF0, 0xFFFF)
114−
115−plane1 :: Gen Int
116−plane1 = oneof [ choose (0x10000, 0x10FFF)
117− , choose (0x11000, 0x11FFF)
118− , choose (0x12000, 0x12FFF)
119− , choose (0x13000, 0x13FFF)
120− , choose (0x1D000, 0x1DFFF)
121− , choose (0x1F000, 0x1FFFF)
122− ]
123−
124−plane2 :: Gen Int
125−plane2 = oneof [ choose (0x20000, 0x20FFF)
126− , choose (0x21000, 0x21FFF)
127− , choose (0x22000, 0x22FFF)
128− , choose (0x23000, 0x23FFF)
129− , choose (0x24000, 0x24FFF)
130− , choose (0x25000, 0x25FFF)
131− , choose (0x26000, 0x26FFF)
132− , choose (0x27000, 0x27FFF)
133− , choose (0x28000, 0x28FFF)
134− , choose (0x29000, 0x29FFF)
135− , choose (0x2A000, 0x2AFFF)
136− , choose (0x2B000, 0x2BFFF)
137− , choose (0x2F000, 0x2FFFF)
138− ]
139−
140−plane14 :: Gen Int
141−plane14 = choose (0xE0000, 0xE0FFF)
142−
143−planes :: [Gen Int]
144−planes = [ascii, plane0, plane1, plane2, plane14]
94+instance Random Word64 where
95+ randomR = integralRandomR
96+ random = randomR (minBound,maxBound)
97+#endif
14598
14699-- For tests that have O(n^2) running times or input sizes, resize
147100-- their inputs to the square root of the originals.
@@ -160,6 +113,20 @@ instance Arbitrary TL.Text where
160113 arbitrary = (TL.fromChunks . map notEmpty) `fmap` smallArbitrary
161114 shrink = map TL.pack . shrink . TL.unpack
162115
116+newtype BigInt = Big Integer
117+ deriving (Eq, Show)
118+
119+instance Arbitrary BigInt where
120+ arbitrary = choose (1::Int,200) >>= \e -> Big <$> choose (10^(e-1),10^e)
121+ shrink (Big a) = [Big (a `div` 2^(l-e)) | e <- shrink l]
122+ where l = truncate (log (fromIntegral a) / log 2 :: Double) :: Integer
123+
124+newtype BigBounded a = BigBounded a
125+ deriving (Eq, Show)
126+
127+instance (Bounded a, Random a, Arbitrary a) => Arbitrary (BigBounded a) where
128+ arbitrary = BigBounded <$> choose (minBound, maxBound)
129+
163130newtype NotEmpty a = NotEmpty { notEmpty :: a }
164131 deriving (Eq, Ord)
165132
@@ -300,6 +267,30 @@ eqP f g s w = eql "orig" (f s) (g t) &&
300267 | a =^= b = True
301268 | otherwise = trace (d ++ ": " ++ show a ++ " /= " ++ show b) False
302269
270+instance Arbitrary FPFormat where
271+ arbitrary = elements [Exponent, Fixed, Generic]
272+
273+newtype Precision a = Precision (Maybe Int)
274+ deriving (Eq, Show)
275+
276+precision :: a -> Precision a -> Maybe Int
277+precision _ (Precision prec) = prec
278+
279+arbitraryPrecision :: Int -> Gen (Precision a)
280+arbitraryPrecision maxDigits = Precision <$> do
281+ n <- choose (-1,maxDigits)
282+ return $ if n == -1
283+ then Nothing
284+ else Just n
285+
286+instance Arbitrary (Precision Float) where
287+ arbitrary = arbitraryPrecision 11
288+ shrink = map Precision . shrink . precision undefined
289+
290+instance Arbitrary (Precision Double) where
291+ arbitrary = arbitraryPrecision 22
292+ shrink = map Precision . shrink . precision undefined
293+
303294-- Work around lack of Show instance for TextEncoding.
304295data Encoding = E String IO.TextEncoding
305296
tests/cabal.config−1
@@ -1,7 +1,6 @@
11-- These flags help to speed up building the test suite.
22
33documentation: False
4−executable-profiling: False
54executable-stripping: False
65flags: developer
76library-profiling: False
tests/literal-rule-test.shnew file+29
@@ -0,0 +1,29 @@
1+#!/bin/bash -e
2+
3+failed=0
4+
5+function check_firings() {
6+ rule=$1
7+ expected=$2
8+ build="ghc -O -ddump-rule-firings LiteralRuleTest.hs"
9+ build="$build -i.. -I../include"
10+ touch LiteralRuleTest.hs
11+ echo -n "Want to see $expected firings of rule $rule... " >&2
12+ firings=$($build 2>&1 | grep "Rule fired: $rule\$" | wc -l)
13+ rm -f LiteralRuleTest.{o.hi}
14+
15+ if [ $firings != $expected ]; then
16+ echo "failed, saw $firings" >&2
17+ failed=1
18+ else
19+ echo "pass" >&2
20+ fi
21+}
22+
23+check_firings "TEXT literal" 8
24+check_firings "TEXT literal UTF8" 7
25+check_firings "TEXT empty literal" 4
26+# This is broken at the moment. "TEXT literal" rule fires instead.
27+#check_firings "TEXT singleton literal" 5
28+
29+exit $failed
tests/text-tests.cabal+4−1
@@ -3,7 +3,7 @@ version: 0.0.0.0
33synopsis: Functional tests for the text package
44description: Functional tests for the text package
55homepage: https://github.com/bos/text
6−license: BSD3
6+license: BSD2
77license-file: ../LICENSE
88author: Jasper Van der Jeugt <jaspervdj@gmail.com>,
99 Bryan O'Sullivan <bos@serpentine.com>,
@@ -51,6 +51,7 @@ executable text-tests
5151 bytestring,
5252 deepseq,
5353 directory,
54+ quickcheck-unicode,
5455 random,
5556 test-framework >= 0.4,
5657 test-framework-hunit >= 0.2,
@@ -113,6 +114,7 @@ library
113114 Data.Text.Internal.Lazy.Search
114115 Data.Text.Internal.Private
115116 Data.Text.Read
117+ Data.Text.Show
116118 Data.Text.Internal.Read
117119 Data.Text.Internal.Search
118120 Data.Text.Unsafe
@@ -134,6 +136,7 @@ library
134136 build-depends:
135137 array,
136138 base == 4.*,
139+ binary,
137140 bytestring,
138141 deepseq,
139142 ghc-prim,
text.cabal+24−6
@@ -1,5 +1,5 @@
11name: text
2−version: 1.1.1.3
2+version: 1.2.2.1
33homepage: https://github.com/bos/text
44bug-reports: https://github.com/bos/text/issues
55synopsis: An efficient packed Unicode text type.
@@ -31,7 +31,7 @@ description:
3131 the @text-icu@ package:
3232 <http://hackage.haskell.org/package/text-icu>
3333
34−license: BSD3
34+license: BSD2
3535license-file: LICENSE
3636author: Bryan O'Sullivan <bos@serpentine.com>
3737maintainer: Bryan O'Sullivan <bos@serpentine.com>
@@ -64,6 +64,11 @@ extra-source-files:
6464 tests/scripts/*.sh
6565 tests/text-tests.cabal
6666
67+flag bytestring-builder
68+ description: Depend on the bytestring-builder package for backwards compatibility.
69+ default: False
70+ manual: False
71+
6772flag developer
6873 description: operate in developer mode
6974 default: False
@@ -72,6 +77,7 @@ flag developer
7277flag integer-simple
7378 description: Use the simple integer library instead of GMP
7479 default: False
80+ manual: False
7581
7682library
7783 c-sources: cbits/cbits.c
@@ -122,16 +128,21 @@ library
122128 Data.Text.Read
123129 Data.Text.Unsafe
124130
131+ other-modules:
132+ Data.Text.Show
133+
125134 build-depends:
126135 array >= 0.3,
127136 base >= 4.2 && < 5,
137+ binary,
128138 deepseq >= 1.1.0.0,
129139 ghc-prim >= 0.2
130140
131− if impl(ghc >= 7.7)
132− build-depends: bytestring >= 0.10.4.0
141+ if flag(bytestring-builder)
142+ build-depends: bytestring >= 0.9 && < 0.10.4,
143+ bytestring-builder >= 0.10.4
133144 else
134− build-depends: bytestring >= 0.9
145+ build-depends: bytestring >= 0.10.4
135146
136147 cpp-options: -DHAVE_DEEPSEQ
137148 ghc-options: -Wall -fwarn-tabs -funbox-strict-fields -O2
@@ -165,15 +176,22 @@ test-suite tests
165176 QuickCheck >= 2.7,
166177 array,
167178 base,
168− bytestring,
179+ binary,
169180 deepseq,
170181 directory,
171182 ghc-prim,
183+ quickcheck-unicode,
172184 random,
173185 test-framework >= 0.4,
174186 test-framework-hunit >= 0.2,
175187 test-framework-quickcheck2 >= 0.2
176188
189+ if flag(bytestring-builder)
190+ build-depends: bytestring >= 0.9 && < 0.10.4,
191+ bytestring-builder >= 0.10.4
192+ else
193+ build-depends: bytestring >= 0.10.4
194+
177195 if flag(integer-simple)
178196 cpp-options: -DINTEGER_SIMPLE
179197 build-depends: integer-simple >= 0.1 && < 0.5