254f27f · streamly · git.iko.soy

git.iko.soy/streamly

Add unlines, unwords, tests for lines/unlines and words/unwords.

commit254f27fde877e5727ddecc97b8fac5a72f3440ca authorPranay Sashank <pranaysashank@gmail.com> dateWed 5 Jun 2019, 11:58 (7 years ago) parent7b216d8

5 files changed+147−23

compare with master

unifiedsplit

examples/HandleIO.hs+3−1 src/Streamly/FileSystem/Handle.hs+16−3 src/Streamly/Mem/Array/Types.hs+45−2 src/Streamly/String.hs+27−11 test/String.hs+56−6

examples/HandleIO.hs+3−1

@@ -2,6 +2,7 @@ import qualified Streamly.Prelude as S

22import qualified Streamly.Fold as FL

33import qualified Streamly.Mem.Array as A

44import qualified Streamly.FileSystem.Handle as FH

5+import qualified Streamly.String as SS

67import Data.Char (ord)

78import System.Environment (getArgs)

@@ -18,7 +19,8 @@ ord' = (fromIntegral . ord)

1819

1920wcl :: Handle -> IO ()

2021wcl src = print =<< (S.length

21− $ FL.splitSuffixBy (== ord' '\n') FL.drain

22+ $ flip SS.foldLines FL.drain

23+ $ SS.decodeChar8

2224 $ FH.read src)

2325

2426grepc :: String -> Handle -> IO ()

src/Streamly/FileSystem/Handle.hs+16−3

@@ -114,6 +114,7 @@ import Streamly.Streams.StreamK.Type (IsStream, mkStream)

114114import qualified Streamly.Mem.Array as A

115115import qualified Streamly.Mem.Array.Types as A hiding (flattenArrays)

116116import qualified Streamly.Prelude as S

117+import qualified Streamly.Streams.StreamD.Type as D

117118

118119-------------------------------------------------------------------------------

119120-- References

@@ -174,10 +175,10 @@ writeArray h Array{..} = withForeignPtr aStart $ \p -> hPutBuf h p aLen

174175-- | @readArraysUpto size h@ reads a stream of arrays from file handle @h@.

175176-- The maximum size of a single array is specified by @size@. The actual size

176177-- read may be less than or equal to @size@.

177−{-# INLINABLE readArraysUpto #-}

178−readArraysUpto :: (IsStream t, MonadIO m)

178+{-# INLINABLE _readArraysUpto #-}

179+_readArraysUpto :: (IsStream t, MonadIO m)

179180 => Int -> Handle -> t m (Array Word8)

180−readArraysUpto size h = go

181+_readArraysUpto size h = go

181182 where

182183 -- XXX use cons/nil instead

183184 go = mkStream $ \_ yld _ stp -> do

@@ -186,6 +187,18 @@ readArraysUpto size h = go

186187 then stp

187188 else yld arr go

188189

190+{-# INLINE_NORMAL readArraysUpto #-}

191+readArraysUpto :: (IsStream t, MonadIO m) => Int -> Handle -> t m (Array Word8)

192+readArraysUpto size h = D.fromStreamD (D.Stream step ())

193+ where

194+ {-# INLINE_LATE step #-}

195+ step _ _ = do

196+ arr <- liftIO $ readArrayUpto size h

197+ return $

198+ case A.length arr of

199+ 0 -> D.Stop

200+ _ -> D.Yield arr ()

201+

189202-- XXX read 'Array a' instead of Word8

190203--

191204-- | @readArrays h@ reads a stream of arrays from file handle @h@.

src/Streamly/Mem/Array/Types.hs+45−2

@@ -6,7 +6,7 @@

66{-# LANGUAGE ScopedTypeVariables #-}

77{-# LANGUAGE TypeFamilies #-}

88{-# LANGUAGE UnboxedTuples #-}

9−

9+{-# LANGUAGE FlexibleContexts #-}

1010#include "inline.hs"

1111

1212-- |

@@ -57,6 +57,8 @@ module Streamly.Mem.Array.Types

5757 , mkChunkSize

5858 , mkChunkSizeKB

5959 , reallocDouble

60+

61+ , unlines

6062 )

6163where

6264

@@ -73,7 +75,7 @@ import Foreign.ForeignPtr

7375import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)

7476import Foreign.Ptr (plusPtr, minusPtr, castPtr)

7577import Foreign.Storable (Storable(..))

76−import Prelude hiding (length, foldr, read)

78+import Prelude hiding (length, foldr, read, unlines, unwords)

7779import Text.Read (readPrec, readListPrec, readListPrecDefault)

7880

7981import GHC.Base (Addr#, realWorld#)

@@ -344,6 +346,22 @@ toStreamD Array{..} =

344346 return r

345347 return $ D.Yield x (p `plusPtr` (sizeOf (undefined :: a)))

346348

349+

350+{-# INLINE toStreamK #-}

351+toStreamK :: forall t m a. (K.IsStream t, Monad m, Storable a) => Array a -> t m a

352+toStreamK Array{..} =

353+ let p = unsafeForeignPtrToPtr aStart

354+ in go p

355+ where

356+ go p | p == aEnd = K.nil

357+ | otherwise =

358+ -- See Note in toStreamD.

359+ let !x = unsafeInlineIO $ do

360+ r <- peek p

361+ touchForeignPtr aStart

362+ return r

363+ in x `K.cons` go (p `plusPtr` (sizeOf (undefined :: a)))

364+

347365{-# INLINE_NORMAL toStreamDRev #-}

348366toStreamDRev :: forall m a. (Monad m, Storable a) => Array a -> D.Stream m a

349367toStreamDRev Array{..} =

@@ -665,3 +683,28 @@ mkChunkSizeKB n = mkChunkSize (n * k)

665683-- bytes, so that the actual allocation is 32KB.

666684defaultChunkSize :: Int

667685defaultChunkSize = mkChunkSizeKB 32

686+

687+{-# INLINE_NORMAL unlines #-}

688+unlines :: MonadIO m => D.Stream m (Array Char) -> D.Stream m Char

689+unlines (D.Stream step state) = D.Stream step' (OuterLoop state)

690+ where

691+ {-# INLINE_LATE step' #-}

692+ step' gst (OuterLoop st) = do

693+ r <- step (adaptState gst) st

694+ return $ case r of

695+ D.Yield Array{..} s ->

696+ let p = unsafeForeignPtrToPtr aStart

697+ in D.Skip (InnerLoop s aStart p aEnd)

698+ D.Skip s -> D.Skip (OuterLoop s)

699+ D.Stop -> D.Stop

700+

701+ step' _ (InnerLoop st _ p end) | p == end =

702+ return $ D.Yield '\n' $ OuterLoop st

703+

704+ step' _ (InnerLoop st startf p end) = do

705+ x <- liftIO $ do

706+ r <- peek p

707+ touchForeignPtr startf

708+ return r

709+ return $ D.Yield x (InnerLoop st startf

710+ (p `plusPtr` (sizeOf (undefined :: Char))) end)

src/Streamly/String.hs+27−11

@@ -1,3 +1,4 @@

1+{-# LANGUAGE FlexibleContexts #-}

12-- |

23-- Module : Streamly.String

@@ -52,23 +53,25 @@ module Streamly.String

5253 , foldLines

5354 , foldWords

5455 , lines

55− , words{-

56+ , words

5657 , unlines

57− , unwords-}

58+ , unwords

5859 )

5960where

6061

6162import Control.Monad.IO.Class (MonadIO)

62−import Data.Char (isSpace, ord)

63+import Data.Char (ord)

6364import Data.Word (Word8)

6465import GHC.Base (unsafeChr)

65−import Streamly (IsStream)

66+import Streamly (IsStream, MonadAsync)

6667import Prelude hiding (String, lines, words, unlines, unwords)

6768import Streamly.Fold (Fold)

6869import Streamly.Mem.Array (Array, toArray)

6970

7071import qualified Streamly.Prelude as S

7172import qualified Streamly.Fold as FL

73+import qualified Streamly.Mem.Array.Types as A (unlines)

74+import qualified Streamly.Mem.Array as A

7275import qualified Streamly.Streams.StreamD as D

7376

7477-- type String = List Char

@@ -112,12 +115,13 @@ encodeChar8Unchecked :: (IsStream t, Monad m) => t m Char -> t m Word8

112115encodeChar8Unchecked = S.map (fromIntegral . ord)

113116

114117-- | Decode a UTF-8 encoded bytestream to a stream of Unicode characters.

118+-- The incoming stream is truncated if an invalid codepoint is encountered.

115119{-# INLINE decodeUtf8 #-}

116120decodeUtf8 :: (Monad m, IsStream t) => t m Word8 -> t m Char

117121decodeUtf8 = D.fromStreamD . D.decodeUtf8 . D.toStreamD

118122

119−{-# INLINE encodeUtf8 #-}

120123-- | Encode a stream of Unicode characters to a UTF-8 encoded bytestream.

124+{-# INLINE encodeUtf8 #-}

121125encodeUtf8 :: (Monad m, IsStream t) => t m Char -> t m Word8

122126encodeUtf8 = D.fromStreamD . D.encodeUtf8 . D.toStreamD

123127

@@ -161,6 +165,18 @@ foldLines = flip (FL.splitSuffixBy (== '\n'))

161165foldWords :: (Monad m, IsStream t) => t m Char -> Fold m Char b -> t m b

162166foldWords = flip (FL.wordsBy isSpace)

163167

168+foreign import ccall unsafe "u_iswspace"

169+ iswspace :: Int -> Int

170+

171+-- Code copied from base/Data.Char to INLINE it

172+{-# INLINE isSpace #-}

173+isSpace :: Char -> Bool

174+isSpace c

175+ | uc <= 0x377 = uc == 32 || uc - 0x9 <= 4 || uc == 0xa0

176+ | otherwise = iswspace (ord c) /= 0

177+ where

178+ uc = fromIntegral (ord c) :: Word

179+

164180{-# INLINE lines #-}

165181lines :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)

166182lines = FL.splitSuffixBy (== '\n') toArray

@@ -169,10 +185,10 @@ lines = FL.splitSuffixBy (== '\n') toArray

169185words :: (MonadIO m, IsStream t) => t m Char -> t m (Array Char)

170186words = FL.wordsBy isSpace toArray

171187

172−{-

173−unlines :: IsStream t => t m (Array Char) -> t m Char

174−unlines = undefined

188+{-# INLINE unlines #-}

189+unlines :: (MonadIO m, IsStream t) => t m (Array Char) -> t m Char

190+unlines = D.fromStreamD . A.unlines . D.toStreamD

175191

176−unwords :: IsStream t => t m (Array Char) -> t m Char

177−unwords = undefined

178−-}

192+{-# INLINE unwords #-}

193+unwords :: (MonadAsync m, IsStream t) => t m (Array Char) -> t m Char

194+unwords = A.flattenArrays . (S.intersperseM (return $ A.fromList " "))

test/String.hs+56−6

@@ -3,14 +3,15 @@

44module Main (main) where

6−import Test.Hspec.QuickCheck

7−import Test.QuickCheck (Property, forAll, Gen, listOf, arbitraryUnicodeChar)

8−import Test.QuickCheck.Monadic (monadicIO, assert)

6+import Test.Hspec.QuickCheck

7+import Test.QuickCheck (Property, forAll, Gen, listOf, arbitraryUnicodeChar)

8+import Test.QuickCheck.Monadic (run, monadicIO, assert)

10−import Test.Hspec as H

10+import Test.Hspec as H

1111

12−import qualified Streamly.String as SS

12+import qualified Streamly.Mem.Array as A

1313import qualified Streamly.Prelude as S

14+import qualified Streamly.String as SS

1415

1516-- Coverage build takes too long with default number of tests

1617maxTestCount :: Int

@@ -32,10 +33,59 @@ propDecodeEncodeId =

3233 chrs <- S.toList $ SS.decodeUtf8 wrds

3334 assert (chrs == list)

3435

36+testLines :: Property

37+testLines =

38+ forAll genUnicode $ \list ->

39+ monadicIO $ do

40+ xs <- S.toList

41+ $ S.map A.toList

42+ $ SS.lines

43+ $ S.fromList list

44+ assert (xs == lines list)

45+

46+testWords :: Property

47+testWords =

48+ forAll genUnicode $ \list ->

49+ monadicIO $ do

50+ xs <- S.toList

51+ $ S.map A.toList

52+ $ SS.words

53+ $ S.fromList list

54+ assert (xs == words list)

55+

56+testUnlines :: Property

57+testUnlines =

58+ forAll genUnicode $ \list ->

59+ monadicIO $ do

60+ xs <- S.toList

61+ $ SS.unlines

62+ $ SS.lines

63+ $ S.fromList list

64+ assert (xs == unlines (lines list))

65+

66+testUnwords :: Property

67+testUnwords =

68+ forAll genUnicode $ \list ->

69+ monadicIO $ do

70+ xs <- run

71+ $ S.toList

72+ $ SS.unwords

73+ $ SS.words

74+ $ S.fromList list

75+ assert (xs == unwords (words list))

76+

3577main :: IO ()

3678main = hspec

3779 $ H.parallel

38− $ modifyMaxSuccess (const maxTestCount)

80+ $ modifyMaxSuccess (const 1000)

3981 $ do

4082 describe "UTF8 - Encoding / Decoding" $ do

4183 prop "decodeUtf8 . encodeUtf8 == id" $ propDecodeEncodeId

84+ prop "Streamly.String.lines == Prelude.lines" $ testLines

85+ prop "Streamly.String.words == Prelude.words" $ testWords

86+ prop

87+ "Streamly.String.unlines . Streamly.String.lines == unlines . lines"

88+ $ testUnlines

89+ prop

90+ "Streamly.String.unwords . Streamly.String.words == unwords . words"

91+ $ testUnwords