2d18f9e · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Expose decodeUtf8Chunks

2d18f9eHarendra Kumar4 years ago

benchmark/Streamly/Benchmark/Data/Array/Stream.hs+1−1

compare with master

unifiedsplit

@@ -123,7 +123,7 @@ inspect $ 'toChunksCountBytes `hasNoType` ''Step

123

124toChunksDecodeUtf8Arrays :: Handle -> IO ()

125toChunksDecodeUtf8Arrays =

126− Stream.drain . Unicode.decodeUtf8Arrays . Handle.readChunks

126+ Stream.drain . Unicode.decodeUtf8Chunks . Handle.readChunks

127

128#ifdef INSPECTION

129inspect $ hasNoTypeClasses 'toChunksDecodeUtf8Arrays

benchmark/Streamly/Benchmark/Unicode/Stream.hs+1−1

@@ -56,7 +56,7 @@ copyCodecUtf8ArraysLenient :: Handle -> Handle -> IO ()

56copyCodecUtf8ArraysLenient inh outh =

57 Stream.fold (Handle.write outh)

58 $ Unicode.encodeUtf8'

59− $ Unicode.decodeUtf8Arrays

59+ $ Unicode.decodeUtf8Chunks

60 $ Handle.readChunks inh

62#ifdef INSPECTION

core/src/Streamly/Internal/Unicode/Stream.hs+18−37

@@ -30,9 +30,9 @@ module Streamly.Internal.Unicode.Stream

30 , resumeDecodeUtf8Either

32 -- ** UTF-8 Array Stream Decoding

33− , decodeUtf8Arrays

33+ , decodeUtf8Chunks

34− , decodeUtf8Arrays'

34+ , decodeUtf8Chunks'

35− , decodeUtf8Arrays_

35+ , decodeUtf8Chunks_

37 -- * Elimination (Encoding)

38 -- ** Latin1 Encoding

@@ -68,9 +68,6 @@ module Streamly.Internal.Unicode.Stream

68 , encodeUtf8D_

69 , decodeUtf8EitherD

70 , resumeDecodeUtf8EitherD

71− , decodeUtf8ArraysD

72− , decodeUtf8ArraysD'

73− , decodeUtf8ArraysD_

75 -- * Decoding String Literals

72 -- * Decoding String Literals

76 , fromStr#

73 , fromStr#

@@ -804,48 +801,32 @@ decodeUtf8ArraysWithD cfm (D.Stream step state) =

804 step' _ _ (YAndC c s) = return $ Yield c s

801 step' _ _ (YAndC c s) = return $ Yield c s

805 step' _ _ D = return Stop

802 step' _ _ D = return Stop

806

803

807−{-# INLINE decodeUtf8ArraysD #-}

804+-- | Like 'decodeUtf8' but for a chunked stream. It may be slightly faster than

808−decodeUtf8ArraysD ::

805+-- flattening the stream and then decoding with 'decodeUtf8'.

806+{-# INLINE decodeUtf8Chunks #-}

807+decodeUtf8Chunks ::

809 MonadIO m

808 MonadIO m

810 => D.Stream m (Array Word8)

809 => D.Stream m (Array Word8)

811 -> D.Stream m Char

810 -> D.Stream m Char

812−decodeUtf8ArraysD = decodeUtf8ArraysWithD TransliterateCodingFailure

811+decodeUtf8Chunks = decodeUtf8ArraysWithD TransliterateCodingFailure

813

812

814−-- |

813+-- | Like 'decodeUtf8\'' but for a chunked stream. It may be slightly faster

815−--

814+-- than flattening the stream and then decoding with 'decodeUtf8\''.

816−-- /Pre-release/

815+{-# INLINE decodeUtf8Chunks' #-}

817−{-# INLINE decodeUtf8Arrays #-}

816+decodeUtf8Chunks' ::

818−decodeUtf8Arrays :: MonadIO m => Stream m (Array Word8) -> Stream m Char

819−decodeUtf8Arrays = decodeUtf8ArraysD

820−

821−{-# INLINE decodeUtf8ArraysD' #-}

822−decodeUtf8ArraysD' ::

823 MonadIO m

817 MonadIO m

824 => D.Stream m (Array Word8)

818 => D.Stream m (Array Word8)

825 -> D.Stream m Char

819 -> D.Stream m Char

826−decodeUtf8ArraysD' = decodeUtf8ArraysWithD ErrorOnCodingFailure

820+decodeUtf8Chunks' = decodeUtf8ArraysWithD ErrorOnCodingFailure

827−

828−-- |

829−--

830−-- /Pre-release/

831−{-# INLINE decodeUtf8Arrays' #-}

832−decodeUtf8Arrays' :: MonadIO m => Stream m (Array Word8) -> Stream m Char

833−decodeUtf8Arrays' = decodeUtf8ArraysD'

834

821

835−{-# INLINE decodeUtf8ArraysD_ #-}

822+-- | Like 'decodeUtf8_' but for a chunked stream. It may be slightly faster

836−decodeUtf8ArraysD_ ::

823+-- than flattening the stream and then decoding with 'decodeUtf8_'.

824+{-# INLINE decodeUtf8Chunks_ #-}

825+decodeUtf8Chunks_ ::

837 MonadIO m

826 MonadIO m

838 => D.Stream m (Array Word8)

827 => D.Stream m (Array Word8)

839 -> D.Stream m Char

828 -> D.Stream m Char

840−decodeUtf8ArraysD_ = decodeUtf8ArraysWithD DropOnCodingFailure

829+decodeUtf8Chunks_ = decodeUtf8ArraysWithD DropOnCodingFailure

841−

842−-- |

843−--

844−-- /Pre-release/

845−{-# INLINE decodeUtf8Arrays_ #-}

846−decodeUtf8Arrays_ ::

847− MonadIO m => Stream m (Array Word8) -> Stream m Char

848−decodeUtf8Arrays_ = decodeUtf8ArraysD_

849

830

850-------------------------------------------------------------------------------

831-------------------------------------------------------------------------------

851-- Encoding Unicode (UTF-8) Characters

832-- Encoding Unicode (UTF-8) Characters

core/src/Streamly/Unicode/Stream.hs+1

@@ -79,6 +79,7 @@ module Streamly.Unicode.Stream

79 decodeLatin1

80 , decodeUtf8

81 , decodeUtf8'

82+ , decodeUtf8Chunks

83 -- * Elimination (Encoding)

84 -- * Elimination (Encoding)

84 , encodeLatin1

85 , encodeLatin1

test/Streamly/Test/Unicode/Stream.hs+1−1

@@ -68,7 +68,7 @@ propDecodeEncodeIdArrays =

68 forAll genUnicode $ \list ->

69 monadicIO $ do

70 let wrds = Stream.arraysOf 8 $ SS.encodeUtf8' $ Stream.fromList list

71− chrs <- Stream.toList $ IUS.decodeUtf8Arrays wrds

71+ chrs <- Stream.toList $ IUS.decodeUtf8Chunks wrds

72 assert (chrs == list)

74unicodeTestData :: [Char]