44e3cb3 · streamly · git.iko.soy

git.iko.soy/streamly

hide char array string ops from Unicode stream module

commit44e3cb36e948fa21ba66b2a8b74e4335764943e5 authorHarendra Kumar <harendra.kumar@gmail.com> dateSun 3 Nov 2019, 23:54 (7 years ago) parentab6587b

5 files changed+33−45

compare with master

unifiedsplit

examples/WordCount.hs+12−9 src/Streamly/Benchmark/FileIO/Array.hs+3−2 src/Streamly/Benchmark/FileIO/Stream.hs+6−5 src/Streamly/Data/Unicode/Stream.hs+4−22 test/String.hs+8−7

examples/WordCount.hs+12−9

@@ -36,6 +36,9 @@ import Data.Word (Word8)

3636import GHC.Conc (numCapabilities)

3737import System.Environment (getArgs)

3838import System.IO (Handle, openFile, IOMode(..))

39+import Streamly.Internal.Data.Unicode.Stream

40+ (DecodeState, DecodeError(..), CodePoint, decodeUtf8Either,

41+ resumeDecodeUtf8Either)

3942

4043import qualified Streamly as S

4144import qualified Streamly.Data.Unicode.Stream as S

@@ -239,7 +242,7 @@ resetHeaderOnNewChar counts = do

239242-- Manipulating the trailer

240243-------------------------------------------------------------------------------

241244

242−setTrailer :: V.IOVector Int -> S.DecodeState -> S.CodePoint -> IO ()

245+setTrailer :: V.IOVector Int -> DecodeState -> CodePoint -> IO ()

243246setTrailer counts st cp = do

244247 writeField counts TrailerState (fromIntegral st)

245248 writeField counts TrailerCodePoint cp

@@ -257,7 +260,7 @@ resetTrailerOnNewChar counts = do

257260-------------------------------------------------------------------------------

258261

259262{-# INLINE countChar #-}

260−countChar :: V.IOVector Int -> Either S.DecodeError Char -> IO ()

263+countChar :: V.IOVector Int -> Either DecodeError Char -> IO ()

261264countChar counts inp =

262265 case inp of

263266 Right ch -> do

@@ -272,7 +275,7 @@ countChar counts inp =

272275 wasSpace <- readField counts WasSpace

273276 when (wasSpace /= 0) $ modifyField counts WordCount (+ 1)

274277 accountChar counts False

275− Left (S.DecodeError st cp) -> do

278+ Left (DecodeError st cp) -> do

276279 hdone <- readField counts HeaderDone

277280 if hdone == 0

278281 then do

@@ -309,7 +312,7 @@ _wc_mwl_parserial :: Handle -> IO (V.IOVector Int)

309312_wc_mwl_parserial src = do

310313 counts <- newCounts

311314 S.mapM_ (countChar counts)

312− $ S.decodeUtf8Either

315+ $ decodeUtf8Either

313316 $ S.unfold FH.read src

314317 return counts

315318

@@ -348,7 +351,7 @@ _wc_mwl_serial src = print =<< (

348351reconstructChar :: Int

349352 -> V.IOVector Int

350353 -> V.IOVector Int

351− -> IO (S.SerialT IO (Either S.DecodeError Char))

354+ -> IO (S.SerialT IO (Either DecodeError Char))

352355reconstructChar hdrCnt v1 v2 = do

353356 when (hdrCnt > 3 || hdrCnt < 0) $ error "reconstructChar: hdrCnt > 3"

354357 stream1 <-

@@ -372,7 +375,7 @@ reconstructChar hdrCnt v1 v2 = do

372375

373376 state <- readField v1 TrailerState

374377 cp <- readField v1 TrailerCodePoint

375− return $ S.resumeDecodeUtf8Either (fromIntegral state) cp stream3

378+ return $ resumeDecodeUtf8Either (fromIntegral state) cp stream3

376379

377380getHdrChar :: V.IOVector Int -> IO (Maybe Int)

378381getHdrChar v = do

@@ -543,7 +546,7 @@ addCounts v1 v2 = do

543546 writeField v1 TrailerState trailerState2

544547 writeField v1 TrailerCodePoint trailerCodePoint2

545548 return v1

546− Left (S.DecodeError st cp) -> do

549+ Left (DecodeError st cp) -> do

547550 -- if header was incomplete it may result

548551 -- in partially decoded char to be written

549552 -- as trailer. Check if the last error is

@@ -554,7 +557,7 @@ addCounts v1 v2 = do

554557 Nothing -> (st, cp)

555558 Just lst -> case lst of

556559 Right _ -> error "addCounts: Bug"

557− Left (S.DecodeError st1 cp1) -> (st1, cp1)

560+ Left (DecodeError st1 cp1) -> (st1, cp1)

558561 if hdone2 == 0 && st' /= 12

559562 then do

560563 -- all elements before the last one must be errors

@@ -593,7 +596,7 @@ countArray :: A.Array Word8 -> IO (V.IOVector Int)

593596countArray src = do

594597 counts <- newCounts

595598 S.mapM_ (countChar counts)

596− $ S.decodeUtf8Either

599+ $ decodeUtf8Either

597600 $ S.unfold A.read src

598601 return counts

599602

src/Streamly/Benchmark/FileIO/Array.hs+3−2

@@ -46,6 +46,7 @@ import qualified Streamly.FileSystem.Handle as FH

4646import qualified Streamly.Memory.Array as A

4747import qualified Streamly.Prelude as S

4848import qualified Streamly.Data.Unicode.Stream as SS

49+import qualified Streamly.Internal.Data.Unicode.Stream as IUS

4950

5051import qualified Streamly.Internal.FileSystem.Handle as IFH

5152import qualified Streamly.Internal.Memory.Array as IA

@@ -214,7 +215,7 @@ inspect $ hasNoTypeClassesExcept 'wordsUnwordsCopy [''Storable]

214215decodeUtf8Lenient :: Handle -> IO ()

215216decodeUtf8Lenient inh =

216217 S.drain

217− $ SS.decodeUtf8ArraysLenient

218+ $ IUS.decodeUtf8ArraysLenient

218219 $ IFH.toStreamArraysOf (1024*1024) inh

219220

220221#ifdef INSPECTION

@@ -230,7 +231,7 @@ copyCodecUtf8Lenient :: Handle -> Handle -> IO ()

230231copyCodecUtf8Lenient inh outh =

231232 S.fold (FH.write outh)

232233 $ SS.encodeUtf8

233− $ SS.decodeUtf8ArraysLenient

234+ $ IUS.decodeUtf8ArraysLenient

234235 $ IFH.toStreamArraysOf (1024*1024) inh

235236

236237#ifdef INSPECTION

src/Streamly/Benchmark/FileIO/Stream.hs+6−5

@@ -78,6 +78,7 @@ import qualified Streamly.Prelude as S

7878import qualified Streamly.Data.Fold as FL

7979-- import qualified Streamly.Internal.Data.Fold as IFL

8080import qualified Streamly.Data.Unicode.Stream as SS

81+import qualified Streamly.Internal.Data.Unicode.Stream as IUS

8182import qualified Streamly.Internal.Data.Unfold as IUF

8283import qualified Streamly.Internal.Prelude as IP

8384import qualified Streamly.Streams.StreamD as D

@@ -431,8 +432,8 @@ linesUnlinesArrayCharCopy :: Handle -> Handle -> IO ()

431432linesUnlinesArrayCharCopy inh outh =

432433 S.fold (FH.write outh)

433434 $ SS.encodeChar8

434− $ SS.unlines

435− $ SS.lines

435+ $ IUS.unlines

436+ $ IUS.lines

436437 $ SS.decodeChar8

437438 $ S.unfold FH.read inh

438439

@@ -525,8 +526,8 @@ wordsUnwordsCharArrayCopy :: Handle -> Handle -> IO ()

525526wordsUnwordsCharArrayCopy inh outh =

526527 S.fold (FH.write outh)

527528 $ SS.encodeChar8

528− $ SS.unwords

529− $ SS.words

529+ $ IUS.unwords

530+ $ IUS.words

530531 $ SS.decodeChar8

531532 $ S.unfold FH.read inh

532533

@@ -597,7 +598,7 @@ inspect $ hasNoTypeClasses 'splitOnSeq

597598splitOnSeqUtf8 :: String -> Handle -> IO Int

598599splitOnSeqUtf8 str inh =

599600 (S.length $ IP.splitOnSeq (A.fromList str) FL.drain

600− $ SS.decodeUtf8ArraysLenient

601+ $ IUS.decodeUtf8ArraysLenient

601602 $ IFH.toStreamArrays inh) -- >>= print

602603

603604-- | Split on suffix sequence.

src/Streamly/Data/Unicode/Stream.hs+4−22

@@ -46,40 +46,22 @@ module Streamly.Data.Unicode.Stream

4646 decodeChar8

4747 , decodeUtf8

4848 , decodeUtf8Lenient

49− , D.DecodeError(..)

50− , D.DecodeState

51− , D.CodePoint

52− , decodeUtf8Either

53− , resumeDecodeUtf8Either

54− , decodeUtf8Arrays

55− , decodeUtf8ArraysLenient

5649

5750 -- * Elimination (Encoding)

5851 , encodeChar8

5952 , encodeChar8Unchecked

6053 , encodeUtf8

61−{-

62− -- * Unicode aware operations

63− , toCaseFold

64− , toLower

65− , toUpper

66− , toTitle

67−

54+ {-

6855 -- * Operations on character strings

6956 , strip -- (dropAround isSpace)

70− , stripEnd-}

71− -- * Transformation

57+ , stripEnd

7258 , stripStart

59+ -}

60+ -- * Transformation

7361 , foldLines

7462 , foldWords

7563 , unfoldLines

7664 , unfoldWords

77−

78− -- * Streams of Strings

79− , lines

80− , words

81− , unlines

82− , unwords

8365 )

8466where

8567

test/String.hs+8−7

@@ -16,6 +16,7 @@ import qualified Streamly.Memory.Array as A

1616import qualified Streamly.Internal.Memory.ArrayStream as AS

1717import qualified Streamly.Prelude as S

1818import qualified Streamly.Data.Unicode.Stream as SS

19+import qualified Streamly.Internal.Data.Unicode.Stream as IUS

1920

2021-- Coverage build takes too long with default number of tests

2122{-

@@ -56,7 +57,7 @@ propDecodeEncodeIdArrays =

5657 forAll genUnicode $ \list ->

5758 monadicIO $ do

5859 let wrds = SS.encodeUtf8 $ S.fromList list

59− chrs <- S.toList $ SS.decodeUtf8ArraysLenient

60+ chrs <- S.toList $ IUS.decodeUtf8ArraysLenient

6061 (S.fold A.write wrds)

6162 assert (chrs == list)

6263

@@ -66,7 +67,7 @@ testLines =

6667 monadicIO $ do

6768 xs <- S.toList

6869 $ S.map A.toList

69− $ SS.lines

70+ $ IUS.lines

7071 $ S.fromList list

7172 assert (xs == lines list)

7273

@@ -87,7 +88,7 @@ testWords =

8788 monadicIO $ do

8889 xs <- S.toList

8990 $ S.map A.toList

90− $ SS.words

91+ $ IUS.words

9192 $ S.fromList list

9293 assert (xs == words list)

9394

@@ -96,8 +97,8 @@ testUnlines =

9697 forAll genUnicode $ \list ->

9798 monadicIO $ do

9899 xs <- S.toList

99− $ SS.unlines

100− $ SS.lines

100+ $ IUS.unlines

101+ $ IUS.lines

101102 $ S.fromList list

102103 assert (xs == unlines (lines list))

103104

@@ -107,8 +108,8 @@ testUnwords =

107108 monadicIO $ do

108109 xs <- run

109110 $ S.toList

110− $ SS.unwords

111− $ SS.words

111+ $ IUS.unwords

112+ $ IUS.words

112113 $ S.fromList list

113114 assert (xs == unwords (words list))

114115