on master help sign in
Extracted IO related common functions into a new module
commitc9232b0e25d895d268eb9f38a6b1fa0e01b75dc6 authorAdithya Kumar <adithya@composewell.com> dateTue 27 Jul 2021, 00:07 (5 years ago) parent5283713
17 files changed+152−120
benchmark/Streamly/Benchmark/FileSystem/Handle/ReadWrite.hs+1−1 src/Streamly/Internal/Data/Array/Foreign.hs+3−2 src/Streamly/Internal/Data/Array/Foreign/Mut/Type.hs+37−61 src/Streamly/Internal/Data/Array/Foreign/Type.hs+6−14 src/Streamly/Internal/Data/Array/Stream/Foreign.hs+6−4 src/Streamly/Internal/Data/Stream/StreamD/Transform.hs+2−1 src/Streamly/Internal/FileSystem/FD.hs+3−2 src/Streamly/Internal/FileSystem/File.hs+2−2 src/Streamly/Internal/FileSystem/Handle.hs+2−1 src/Streamly/Internal/Network/Inet/TCP.hs+2−1 src/Streamly/Internal/Network/Socket.hs+12−11 src/Streamly/Internal/Ring/Foreign.hs+13−11 src/Streamly/Internal/System/IO.hs+52 src/Streamly/Internal/Unicode/Stream.hs+2−1 streamly.cabal+1 test/Streamly/Test/Network/Inet/TCP.hs+5−5 test/Streamly/Test/Network/Socket.hs+3−3
benchmark/Streamly/Benchmark/FileSystem/Handle/ReadWrite.hs+1−1
@@ -24,7 +24,7 @@ where
2424
2525import System.IO (Handle)
2626import Prelude hiding (last, length)
27−import Streamly.Internal.Data.Array.Foreign.Type (defaultChunkSize)
27+import Streamly.Internal.System.IO (defaultChunkSize)
2828
2929import qualified Streamly.FileSystem.Handle as FH
3030import qualified Streamly.Internal.Data.Unfold as IUF
src/Streamly/Internal/Data/Array/Foreign.hs+3−2
@@ -144,6 +144,7 @@ import Streamly.Internal.Data.Producer.Type (Producer)
144144import Streamly.Internal.Data.Stream.Serial (SerialT)
145145import Streamly.Internal.Data.Tuple.Strict (Tuple3'(..))
146146import Streamly.Internal.Data.Unfold.Type (Unfold(..))
147+import Streamly.Internal.System.IO (unsafeInlineIO)
147148
148149import qualified Streamly.Internal.Data.Array.Foreign.Mut.Type as MA
149150import qualified Streamly.Internal.Data.Array.Foreign.Type as A
@@ -236,7 +237,7 @@ unsafeRead = Unfold step inject
236237 --
237238 -- This should be safe as the array contents are guaranteed to be
238239 -- evaluated/written to before we peek at them.
239− let !x = A.unsafeInlineIO $ do
240+ let !x = unsafeInlineIO $ do
240241 r <- peek (Ptr p)
241242 touch contents
242243 return r
@@ -378,7 +379,7 @@ getIndex :: Storable a => Array a -> Int -> Maybe a
378379getIndex arr i =
379380 if i < 0 || i > length arr - 1
380381 then Nothing
381− else A.unsafeInlineIO $
382+ else unsafeInlineIO $
382383 unsafeWithForeignPtr (aStart arr) $ \p -> Just <$> peekElemOff p i
383384
384385{-
src/Streamly/Internal/Data/Array/Foreign/Mut/Type.hs+37−61
@@ -1,5 +1,3 @@
1−{-# LANGUAGE UnboxedTuples #-}
2−
31-- |
42-- Module : Streamly.Internal.Data.Array.Foreign.Mut.Type
53-- Copyright : (c) 2020 Composewell Technologies
@@ -92,13 +90,10 @@ module Streamly.Internal.Data.Array.Foreign.Mut.Type
9290 , writeChunks
9391
9492 -- * Utilities
95− , defaultChunkSize
96− , mkChunkSize
97− , mkChunkSizeKB
9893 , bytesToElemCount
99− , unsafeInlineIO
10094 , memcpy
10195 , memcmp
96+ , c_memchr
10297 )
10398where
10499
@@ -118,10 +113,10 @@ import Foreign.ForeignPtr (touchForeignPtr)
118113import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
119114import Foreign.Ptr (plusPtr, minusPtr, castPtr, nullPtr)
120115import Foreign.Storable (Storable(..))
121−import GHC.Base (nullAddr#, realWorld#, build)
116+import GHC.Base (nullAddr#, build)
122117import GHC.Exts (IsList, IsString(..))
123118import GHC.ForeignPtr (ForeignPtr(..))
124−import GHC.IO (IO(IO), unsafePerformIO)
119+import GHC.IO (unsafePerformIO)
125120import GHC.Ptr (Ptr(..))
126121
127122import Streamly.Internal.BaseCompat
@@ -131,6 +126,9 @@ import Streamly.Internal.Data.SVar (adaptState)
131126import Streamly.Internal.Data.Unfold.Type (Unfold(..))
132127import Text.Read (readPrec, readListPrec, readListPrecDefault)
133128
129+import Streamly.Internal.System.IO
130+ (defaultChunkSize, mkChunkSize, unsafeInlineIO)
131+
134132#ifdef DEVBUILD
135133import qualified Data.Foldable as F
136134#endif
@@ -156,6 +154,37 @@ plusForeignPtr (ForeignPtr addr c) (I# d) = ForeignPtr (plusAddr# addr d) c
156154-- Array Data Type
157155-------------------------------------------------------------------------------
158156
157+foreign import ccall unsafe "string.h memcpy" c_memcpy
158+ :: Ptr Word8 -> Ptr Word8 -> CSize -> IO (Ptr Word8)
159+
160+foreign import ccall unsafe "string.h memchr" c_memchr
161+ :: Ptr Word8 -> Word8 -> CSize -> IO (Ptr Word8)
162+
163+foreign import ccall unsafe "string.h memcmp" c_memcmp
164+ :: Ptr Word8 -> Ptr Word8 -> CSize -> IO CInt
165+
166+{-# INLINE bytesToElemCount #-}
167+bytesToElemCount :: Storable a => a -> Int -> Int
168+bytesToElemCount x n =
169+ let elemSize = sizeOf x
170+ in n + elemSize - 1 `div` elemSize
171+
172+-- XXX we are converting Int to CSize
173+memcpy :: Ptr Word8 -> Ptr Word8 -> Int -> IO ()
174+memcpy dst src len = void (c_memcpy dst src (fromIntegral len))
175+
176+-- XXX we are converting Int to CSize
177+-- return True if the memory locations have identical contents
178+{-# INLINE memcmp #-}
179+memcmp :: Ptr Word8 -> Ptr Word8 -> Int -> IO Bool
180+memcmp p1 p2 len = do
181+ r <- c_memcmp p1 p2 (fromIntegral len)
182+ return $ r == 0
183+
184+-------------------------------------------------------------------------------
185+-- Array Data Type
186+-------------------------------------------------------------------------------
187+
159188-- $arrayNotes
160189--
161190-- We can use a 'Storable' constraint in the Array type and the constraint can
@@ -183,59 +212,6 @@ mutableArray ::
183212 ForeignPtr a -> Ptr a -> Ptr a -> Array a
184213mutableArray = Array
185214
186−-------------------------------------------------------------------------------
187−-- Utility functions
188−-------------------------------------------------------------------------------
189−
190−foreign import ccall unsafe "string.h memcpy" c_memcpy
191− :: Ptr Word8 -> Ptr Word8 -> CSize -> IO (Ptr Word8)
192−
193−foreign import ccall unsafe "string.h memchr" c_memchr
194− :: Ptr Word8 -> Word8 -> CSize -> IO (Ptr Word8)
195−
196−-- XXX we are converting Int to CSize
197−memcpy :: Ptr Word8 -> Ptr Word8 -> Int -> IO ()
198−memcpy dst src len = void (c_memcpy dst src (fromIntegral len))
199−
200−foreign import ccall unsafe "string.h memcmp" c_memcmp
201− :: Ptr Word8 -> Ptr Word8 -> CSize -> IO CInt
202−
203−-- XXX we are converting Int to CSize
204−-- return True if the memory locations have identical contents
205−{-# INLINE memcmp #-}
206−memcmp :: Ptr Word8 -> Ptr Word8 -> Int -> IO Bool
207−memcmp p1 p2 len = do
208− r <- c_memcmp p1 p2 (fromIntegral len)
209− return $ r == 0
210−
211−{-# INLINE unsafeInlineIO #-}
212−unsafeInlineIO :: IO a -> a
213−unsafeInlineIO (IO m) = case m realWorld# of (# _, r #) -> r
214−
215−{-# INLINE bytesToElemCount #-}
216−bytesToElemCount :: Storable a => a -> Int -> Int
217−bytesToElemCount x n =
218− let elemSize = sizeOf x
219− in n + elemSize - 1 `div` elemSize
220−
221−
222−-- | GHC memory management allocation header overhead
223−allocOverhead :: Int
224−allocOverhead = 2 * sizeOf (undefined :: Int)
225−
226−mkChunkSize :: Int -> Int
227−mkChunkSize n = let size = n - allocOverhead in max size 0
228−
229−mkChunkSizeKB :: Int -> Int
230−mkChunkSizeKB n = mkChunkSize (n * k)
231− where k = 1024
232−
233−-- | Default maximum buffer size in bytes, for reading from and writing to IO
234−-- devices, the value is 32KB minus GHC allocation overhead, which is a few
235−-- bytes, so that the actual allocation is 32KB.
236−defaultChunkSize :: Int
237−defaultChunkSize = mkChunkSizeKB 32
238−
239215-------------------------------------------------------------------------------
240216-- Construction
241217-------------------------------------------------------------------------------
src/Streamly/Internal/Data/Array/Foreign/Type.hs+6−14
@@ -69,16 +69,6 @@ module Streamly.Internal.Data.Array.Foreign.Type
6969 , bufferChunks
7070 , flattenArrays
7171 , flattenArraysRev
72−
73− -- * Utilities
74− , MA.defaultChunkSize
75− , MA.mkChunkSize
76− , MA.mkChunkSizeKB
77− , MA.unsafeInlineIO
78−
79− , MA.memcpy
80− , MA.memcmp
81− , MA.bytesToElemCount
8272 )
8373where
8474
@@ -109,6 +99,8 @@ import qualified Streamly.Internal.Data.Stream.StreamK.Type as K
10999import qualified Streamly.Internal.Data.Unfold.Type as Unfold
110100import qualified GHC.Exts as Exts
111101
102+import Streamly.Internal.System.IO (unsafeInlineIO, defaultChunkSize)
103+
112104#if __GLASGOW_HASKELL__ < 808
113105import Data.Semigroup (Semigroup(..))
114106#endif
@@ -211,7 +203,7 @@ fromPtr ::
211203 Storable a =>
212204#endif
213205 Int -> Ptr a -> Array a
214−fromPtr n ptr = MA.unsafeInlineIO $ do
206+fromPtr n ptr = unsafeInlineIO $ do
215207 fptr <- newForeignPtr_ ptr
216208 let end = ptr `plusPtr` n
217209 return $ Array
@@ -288,7 +280,7 @@ fromAddr# n addr# = fromPtr n (castPtr $ Ptr addr#)
288280fromCString# :: Addr# -> Array Word8
289281fromCString# addr# = do
290282 let cstr = Ptr addr#
291− len = MA.unsafeInlineIO $ c_strlen cstr
283+ len = unsafeInlineIO $ c_strlen cstr
292284 fromPtr (fromIntegral len) (castPtr cstr)
293285
294286-- | Create an 'Array' from the first N elements of a list. The array is
@@ -328,7 +320,7 @@ fromStreamD str = unsafeFreeze <$> MA.fromStreamD str
328320{-# INLINE bufferChunks #-}
329321bufferChunks :: (MonadIO m, Storable a) =>
330322 D.Stream m a -> m (K.Stream m (Array a))
331−bufferChunks m = D.foldr K.cons K.nil $ arraysOf MA.defaultChunkSize m
323+bufferChunks m = D.foldr K.cons K.nil $ arraysOf defaultChunkSize m
332324
333325-- | @arraysOf n stream@ groups the input stream into a stream of
334326-- arrays of size n.
@@ -381,7 +373,7 @@ unsafeIndexIO arr = MA.unsafeIndexIO (unsafeThaw arr)
381373-- | Return element at the specified index without checking the bounds.
382374{-# INLINE_NORMAL unsafeIndex #-}
383375unsafeIndex :: forall a. Storable a => Array a -> Int -> a
384−unsafeIndex arr i = let !r = MA.unsafeInlineIO $ unsafeIndexIO arr i in r
376+unsafeIndex arr i = let !r = unsafeInlineIO $ unsafeIndexIO arr i in r
385377
386378-- | /O(1)/ Get the byte length of the array.
387379--
src/Streamly/Internal/Data/Array/Stream/Foreign.hs+6−4
@@ -81,6 +81,8 @@ import Streamly.Internal.Data.Parser (ParseError(..))
8181import Streamly.Internal.Data.Stream.Serial (SerialT)
8282import Streamly.Internal.Data.Stream.StreamK.Type (IsStream)
8383import Streamly.Internal.Data.SVar (adaptState, defState)
84+import Streamly.Internal.Data.Array.Foreign.Mut.Type (memcpy, bytesToElemCount)
85+import Streamly.Internal.System.IO (mkChunkSizeKB)
8486
8587import qualified Streamly.Internal.Data.Array.Foreign as A
8688import qualified Streamly.Internal.Data.Array.Foreign as Array
@@ -454,7 +456,7 @@ spliceArraysLenUnsafe len buffered = do
454456 writeArr dst (MA.Array as ae _) =
455457 liftIO $ unsafeWithForeignPtr as $ \src -> do
456458 let count = ae `minusPtr` src
457− A.memcpy (castPtr dst) (castPtr src) count
459+ memcpy (castPtr dst) (castPtr src) count
458460 return $ dst `plusPtr` count
459461
460462{-# INLINE _spliceArrays #-}
@@ -472,7 +474,7 @@ _spliceArrays s = do
472474 writeArr dst (Array as ae) =
473475 liftIO $ unsafeWithForeignPtr as $ \src -> do
474476 let count = ae `minusPtr` src
475− A.memcpy (castPtr dst) (castPtr src) count
477+ memcpy (castPtr dst) (castPtr src) count
476478 return $ dst `plusPtr` count
477479
478480{-# INLINE _spliceArraysBuffered #-}
@@ -487,8 +489,8 @@ _spliceArraysBuffered s = do
487489spliceArraysRealloced :: forall m a. (MonadIO m, Storable a)
488490 => SerialT m (Array a) -> m (Array a)
489491spliceArraysRealloced s = do
490− let idst = liftIO $ MA.newArray (A.bytesToElemCount (undefined :: a)
491− (A.mkChunkSizeKB 4))
492+ let idst = liftIO $ MA.newArray (bytesToElemCount (undefined :: a)
493+ (mkChunkSizeKB 4))
492494
493495 arr <- S.foldlM' MA.spliceWithDoubling idst (S.map A.unsafeThaw s)
494496 liftIO $ A.unsafeFreeze <$> MA.shrinkToFit arr
src/Streamly/Internal/Data/Stream/StreamD/Transform.hs+2−1
@@ -138,6 +138,7 @@ import Streamly.Internal.Data.Pipe.Type (Pipe(..), PipeState(..))
138138import Streamly.Internal.Data.Time.Clock (Clock(Monotonic), getTime)
139139import Streamly.Internal.Data.Time.Units
140140 (TimeUnit64, toRelTime64, diffAbsTime64)
141+import Streamly.Internal.System.IO (defaultChunkSize)
141142
142143import qualified Streamly.Internal.Data.Array.Foreign.Type as A
143144import qualified Streamly.Internal.Data.Fold.Type as FL
@@ -1133,7 +1134,7 @@ reverse' =
11331134 . fromStreamK
11341135 . K.reverse
11351136 . toStreamK
1136− . A.arraysOf A.defaultChunkSize
1137+ . A.arraysOf defaultChunkSize
11371138
11381139------------------------------------------------------------------------------
11391140-- Position Indexing
src/Streamly/Internal/FileSystem/FD.hs+3−2
@@ -130,9 +130,10 @@ import Prelude hiding (read)
130130import qualified GHC.IO.FD as FD
131131import qualified GHC.IO.Device as RawIO
132132
133−import Streamly.Internal.Data.Array.Foreign.Type (Array(..), byteLength, defaultChunkSize, unsafeFreeze)
133+import Streamly.Internal.Data.Array.Foreign.Type
134+ (Array(..), byteLength, unsafeFreeze)
134135import Streamly.Internal.Data.Array.Foreign.Mut.Type (mutableArray)
135−
136+import Streamly.Internal.System.IO (defaultChunkSize)
136137import Streamly.Internal.Data.Stream.Serial (SerialT)
137138import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream)
138139
src/Streamly/Internal/FileSystem/File.hs+2−2
@@ -96,13 +96,13 @@ import qualified System.IO as SIO
9696
9797import Streamly.Internal.Data.Fold.Type (Fold(..))
9898import Streamly.Internal.Data.Unfold.Type (Unfold(..))
99−import Streamly.Internal.Data.Array.Foreign.Type
100− (Array(..), defaultChunkSize, writeNUnsafe)
99+import Streamly.Internal.Data.Array.Foreign.Type (Array(..), writeNUnsafe)
101100import Streamly.Internal.Data.Stream.Serial (SerialT)
102101import Streamly.Internal.Data.Stream.StreamK.Type (IsStream)
103102import Streamly.Internal.Data.SVar (MonadAsync)
104103-- import Streamly.Data.Fold (Fold)
105104-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
105+import Streamly.Internal.System.IO (defaultChunkSize)
106106
107107import qualified Streamly.Internal.Data.Fold.Type as FL
108108import qualified Streamly.Internal.Data.Unfold as UF
src/Streamly/Internal/FileSystem/Handle.hs+2−1
@@ -108,12 +108,13 @@ import Streamly.Data.Fold (Fold)
108108import Streamly.Internal.Data.Fold.Type (Fold2(..))
109109import Streamly.Internal.Data.Unfold.Type (Unfold(..))
110110import Streamly.Internal.Data.Array.Foreign.Type
111− (Array(..), writeNUnsafe, defaultChunkSize , unsafeFreezeWithShrink)
111+ (Array(..), writeNUnsafe, unsafeFreezeWithShrink)
112112import Streamly.Internal.Data.Array.Foreign.Mut.Type (mutableArray)
113113import Streamly.Internal.Data.Stream.Serial (SerialT)
114114import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream)
115115import Streamly.Internal.Data.Array.Stream.Foreign (lpackArraysChunksOf)
116116-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
117+import Streamly.Internal.System.IO (defaultChunkSize)
117118
118119import qualified Streamly.Data.Fold as FL
119120import qualified Streamly.Internal.Data.Fold.Type as FL
src/Streamly/Internal/Network/Inet/TCP.hs+2−1
@@ -103,7 +103,7 @@ import Network.Socket
103103 socket)
104104import Prelude hiding (read)
105105
106−import Streamly.Internal.Data.Array.Foreign.Type (Array(..), defaultChunkSize, writeNUnsafe)
106+import Streamly.Internal.Data.Array.Foreign.Type (Array(..), writeNUnsafe)
107107import Streamly.Internal.Data.Fold.Type (Fold(..))
108108import Streamly.Internal.Data.Stream.IsStream (MonadAsync)
109109import Streamly.Internal.Data.SVar (fork)
@@ -112,6 +112,7 @@ import Streamly.Internal.Data.Stream.StreamK.Type (IsStream)
112112import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))
113113import Streamly.Internal.Data.Unfold.Type (Unfold(..))
114114import Streamly.Internal.Network.Socket (SockSpec(..), accept, connections)
115+import Streamly.Internal.System.IO (defaultChunkSize)
115116
116117import qualified Control.Monad.Catch as MC
117118import qualified Network.Socket as Net
src/Streamly/Internal/Network/Socket.hs+12−11
@@ -95,6 +95,7 @@ import Streamly.Internal.Data.Stream.Serial (SerialT)
9595import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream)
9696import Streamly.Data.Fold (Fold)
9797-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
98+import Streamly.Internal.System.IO (defaultChunkSize)
9899
99100import qualified Streamly.Data.Fold as FL
100101import qualified Streamly.Internal.Data.Unfold as UF
@@ -360,7 +361,7 @@ toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ())
360361-- @since 0.7.0
361362{-# INLINE toChunks #-}
362363toChunks :: (IsStream t, MonadIO m) => Socket -> t m (Array Word8)
363−toChunks = toChunksWithBufferOf A.defaultChunkSize
364+toChunks = toChunksWithBufferOf defaultChunkSize
364365
365366-- | Unfold the tuple @(bufsize, socket)@ into a stream of 'Word8' arrays.
366367-- Read requests to the socket are performed using a buffer of size @bufsize@.
@@ -389,7 +390,7 @@ readChunksWithBufferOf = Unfold step return
389390-- @since 0.7.0
390391{-# INLINE readChunks #-}
391392readChunks :: MonadIO m => Unfold m Socket (Array Word8)
392−readChunks = UF.supplyFirst A.defaultChunkSize readChunksWithBufferOf
393+readChunks = UF.supplyFirst defaultChunkSize readChunksWithBufferOf
393394
394395-------------------------------------------------------------------------------
395396-- Read File to Stream
@@ -412,7 +413,7 @@ readWithBufferOf chunkSize h = A.flattenArrays $ readChunksUpto chunkSize h
412413-- TODO
413414-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
414415--
415−-- > read = 'readByChunks' A.defaultChunkSize
416+-- > read = 'readByChunks' defaultChunkSize
416417-- | Generate a stream of elements of the given type from a socket. The
417418-- stream ends when EOF is encountered.
418419--
@@ -436,7 +437,7 @@ readWithBufferOf = UF.many readChunksWithBufferOf A.read
436437-- @since 0.7.0
437438{-# INLINE read #-}
438439read :: MonadIO m => Unfold m Socket Word8
439−read = UF.supplyFirst A.defaultChunkSize readWithBufferOf
440+read = UF.supplyFirst defaultChunkSize readWithBufferOf
440441
441442-------------------------------------------------------------------------------
442443-- Writing
@@ -495,33 +496,33 @@ putBytesWithBufferOf n h m = putChunks h $ AS.arraysOf n m
495496writeWithBufferOf :: MonadIO m => Int -> Socket -> Fold m Word8 ()
496497writeWithBufferOf n h = FL.chunksOf n (A.writeNUnsafe n) (writeChunks h)
497498
498−-- > write = 'writeWithBufferOf' A.defaultChunkSize
499+-- > write = 'writeWithBufferOf' defaultChunkSize
499500--
500501-- | Write a byte stream to a file handle. Combines the bytes in chunks of size
501−-- up to 'A.defaultChunkSize' before writing. Note that the write behavior
502+-- up to 'defaultChunkSize' before writing. Note that the write behavior
502503-- depends on the 'IOMode' and the current seek position of the handle.
503504--
504505-- @since 0.7.0
505506{-# INLINE putBytes #-}
506507putBytes :: MonadIO m => Socket -> SerialT m Word8 -> m ()
507−putBytes = putBytesWithBufferOf A.defaultChunkSize
508+putBytes = putBytesWithBufferOf defaultChunkSize
508509
509510-- | Write a byte stream to a socket. Accumulates the input in chunks of
510−-- up to 'A.defaultChunkSize' bytes before writing.
511+-- up to 'defaultChunkSize' bytes before writing.
511512--
512513-- @
513−-- write = 'writeWithBufferOf' 'A.defaultChunkSize'
514+-- write = 'writeWithBufferOf' 'defaultChunkSize'
514515-- @
515516--
516517-- @since 0.7.0
517518{-# INLINE write #-}
518519write :: MonadIO m => Socket -> Fold m Word8 ()
519−write = writeWithBufferOf A.defaultChunkSize
520+write = writeWithBufferOf defaultChunkSize
520521
521522{-
522523{-# INLINE write #-}
523524write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
524−write = toHandleWith A.defaultChunkSize
525+write = toHandleWith defaultChunkSize
525526-}
526527
527528-------------------------------------------------------------------------------
src/Streamly/Internal/Ring/Foreign.hs+13−11
@@ -40,6 +40,8 @@ import GHC.Ptr (Ptr(..))
4040import Prelude hiding (length, concat)
4141
4242import Control.Monad.IO.Class (MonadIO(..))
43+import Streamly.Internal.Data.Array.Foreign.Mut.Type (memcmp)
44+import Streamly.Internal.System.IO (unsafeInlineIO)
4345
4446import qualified Streamly.Internal.Data.Array.Foreign.Type as A
4547
@@ -115,7 +117,7 @@ unsafeInsert rb ringHead newVal = do
115117 -- touchForeignPtr (ringStart rb)
116118 return $ advance rb ringHead
117119
118−-- XXX remove all usage of A.unsafeInlineIO
120+-- XXX remove all usage of unsafeInlineIO
119121--
120122-- | Like 'unsafeEqArray' but compares only N bytes instead of entire length of
121123-- the ring buffer. This is unsafe because the ringHead Ptr is not checked to
@@ -123,14 +125,14 @@ unsafeInsert rb ringHead newVal = do
123125{-# INLINE unsafeEqArrayN #-}
124126unsafeEqArrayN :: Ring a -> Ptr a -> A.Array a -> Int -> Bool
125127unsafeEqArrayN Ring{..} rh A.Array{..} n =
126− let !res = A.unsafeInlineIO $ do
128+ let !res = unsafeInlineIO $ do
127129 let rs = unsafeForeignPtrToPtr ringStart
128130 as = unsafeForeignPtrToPtr aStart
129131 assert (aEnd `minusPtr` as >= ringBound `minusPtr` rs) (return ())
130132 let len = ringBound `minusPtr` rh
131− r1 <- A.memcmp (castPtr rh) (castPtr as) (min len n)
133+ r1 <- memcmp (castPtr rh) (castPtr as) (min len n)
132134 r2 <- if n > len
133− then A.memcmp (castPtr rs) (castPtr (as `plusPtr` len))
135+ then memcmp (castPtr rs) (castPtr (as `plusPtr` len))
134136 (min (rh `minusPtr` rs) (n - len))
135137 else return True
136138 -- XXX enable these, check perf impact
@@ -149,14 +151,14 @@ unsafeEqArrayN Ring{..} rh A.Array{..} n =
149151{-# INLINE unsafeEqArray #-}
150152unsafeEqArray :: Ring a -> Ptr a -> A.Array a -> Bool
151153unsafeEqArray Ring{..} rh A.Array{..} =
152− let !res = A.unsafeInlineIO $ do
154+ let !res = unsafeInlineIO $ do
153155 let rs = unsafeForeignPtrToPtr ringStart
154156 let as = unsafeForeignPtrToPtr aStart
155157 assert (aEnd `minusPtr` as >= ringBound `minusPtr` rs)
156158 (return ())
157159 let len = ringBound `minusPtr` rh
158− r1 <- A.memcmp (castPtr rh) (castPtr as) len
159− r2 <- A.memcmp (castPtr rs) (castPtr (as `plusPtr` len))
160+ r1 <- memcmp (castPtr rh) (castPtr as) len
161+ r2 <- memcmp (castPtr rs) (castPtr (as `plusPtr` len))
160162 (rh `minusPtr` rs)
161163 -- XXX enable these, check perf impact
162164 -- touchForeignPtr ringStart
@@ -175,7 +177,7 @@ unsafeEqArray Ring{..} rh A.Array{..} =
175177unsafeFoldRing :: forall a b. Storable a
176178 => Ptr a -> (b -> a -> b) -> b -> Ring a -> b
177179unsafeFoldRing ptr f z Ring{..} =
178− let !res = A.unsafeInlineIO $ withForeignPtr ringStart $ \p ->
180+ let !res = unsafeInlineIO $ withForeignPtr ringStart $ \p ->
179181 go z p ptr
180182 in res
181183 where
@@ -202,7 +204,7 @@ unsafeFoldRingM ptr f z Ring {..} =
202204 go !acc !start !end
203205 | start == end = return acc
204206 | otherwise = do
205− let !x = A.unsafeInlineIO $ peek start
207+ let !x = unsafeInlineIO $ peek start
206208 acc' <- f acc x
207209 go acc' (start `plusPtr` sizeOf (undefined :: a)) end
208210
@@ -220,7 +222,7 @@ unsafeFoldRingFullM rh f z rb@Ring {..} =
220222 withForeignPtrM ringStart $ \_ -> go z rh
221223 where
222224 go !acc !start = do
223− let !x = A.unsafeInlineIO $ peek start
225+ let !x = unsafeInlineIO $ peek start
224226 acc' <- f acc x
225227 let ptr = advance rb start
226228 if ptr == rh
@@ -242,7 +244,7 @@ unsafeFoldRingNM count rh f z rb@Ring {..} =
242244
243245 go 0 acc _ = return acc
244246 go !n !acc !start = do
245− let !x = A.unsafeInlineIO $ peek start
247+ let !x = unsafeInlineIO $ peek start
246248 acc' <- f acc x
247249 let ptr = advance rb start
248250 if ptr == rh || n == 0
src/Streamly/Internal/System/IO.hsnew file+52
@@ -0,0 +1,52 @@
1+{-# LANGUAGE UnboxedTuples #-}
2+
3+-- |
4+-- Module : Streamly.Internal.System.IO
5+-- Copyright : (c) 2020 Composewell Technologies
6+-- License : BSD-3-Clause
7+-- Maintainer : streamly@composewell.com
8+-- Stability : experimental
9+-- Portability : GHC
10+--
11+
12+module Streamly.Internal.System.IO
13+ ( defaultChunkSize
14+ , mkChunkSize
15+ , mkChunkSizeKB
16+ , unsafeInlineIO
17+ )
18+
19+where
20+
21+-------------------------------------------------------------------------------
22+-- Imports
23+-------------------------------------------------------------------------------
24+
25+import Foreign.Storable (Storable(sizeOf))
26+import GHC.Base (realWorld#)
27+import GHC.IO (IO(IO))
28+
29+-------------------------------------------------------------------------------
30+-- API
31+-------------------------------------------------------------------------------
32+
33+{-# INLINE unsafeInlineIO #-}
34+unsafeInlineIO :: IO a -> a
35+unsafeInlineIO (IO m) = case m realWorld# of (# _, r #) -> r
36+
37+-- | GHC memory management allocation header overhead
38+allocOverhead :: Int
39+allocOverhead = 2 * sizeOf (undefined :: Int)
40+
41+mkChunkSize :: Int -> Int
42+mkChunkSize n = let size = n - allocOverhead in max size 0
43+
44+mkChunkSizeKB :: Int -> Int
45+mkChunkSizeKB n = mkChunkSize (n * k)
46+ where k = 1024
47+
48+-- | Default maximum buffer size in bytes, for reading from and writing to IO
49+-- devices, the value is 32KB minus GHC allocation overhead, which is a few
50+-- bytes, so that the actual allocation is 32KB.
51+defaultChunkSize :: Int
52+defaultChunkSize = mkChunkSizeKB 32
src/Streamly/Internal/Unicode/Stream.hs+2−1
@@ -98,6 +98,7 @@ import Streamly.Internal.Data.Stream.IsStream (IsStream)
9898import Streamly.Internal.Data.Stream.Serial (SerialT)
9999import Streamly.Internal.Data.Stream.StreamD (Stream(..), Step (..))
100100import Streamly.Internal.Data.Tuple.Strict (Tuple'(..))
101+import Streamly.Internal.System.IO (unsafeInlineIO)
101102
102103import qualified Streamly.Internal.Data.Unfold as Unfold
103104import qualified Streamly.Internal.Data.Stream.Serial as Serial
@@ -236,7 +237,7 @@ utf8d =
236237-- and without touching the foreign ptr.
237238{-# INLINE_NORMAL unsafePeekElemOff #-}
238239unsafePeekElemOff :: forall a. Storable a => Ptr a -> Int -> a
239−unsafePeekElemOff p i = let !x = A.unsafeInlineIO $ peekElemOff p i in x
240+unsafePeekElemOff p i = let !x = unsafeInlineIO $ peekElemOff p i in x
240241
241242-- decode is split into two separate cases to avoid branching instructions.
242243-- From the higher level flow we already know which case we are in so we can
streamly.cabal+1
@@ -369,6 +369,7 @@ library
369369 , Streamly.Internal.Data.Time.Units
370370 , Streamly.Internal.Data.Time.Clock.Type
371371 , Streamly.Internal.Data.Time.Clock
372+ , Streamly.Internal.System.IO
372373
373374 -- streamly-core-stream
374375 , Streamly.Internal.Data.SVar
test/Streamly/Test/Network/Inet/TCP.hs+5−5
@@ -15,11 +15,11 @@ import Data.Function ((&))
1515import Data.Word (Word8)
1616import Network.Socket (Socket, PortNumber)
1717import Streamly.Internal.Control.Monad (discard)
18+import Streamly.Internal.System.IO (defaultChunkSize)
1819import Streamly.Prelude (SerialT)
1920import Test.QuickCheck (Property)
2021import Test.QuickCheck.Monadic (monadicIO, assert, run)
2122
22−import qualified Streamly.Internal.Data.Array.Foreign.Type as Array
2323import qualified Streamly.Internal.Data.Unfold as Unfold
2424import qualified Streamly.Internal.Network.Inet.TCP as TCP
2525import qualified Streamly.Internal.Network.Socket as Socket
@@ -106,8 +106,8 @@ validateOnPort :: Property
106106validateOnPort = monadicIO $ do
107107 res <- run $ do
108108 ls2 <-
109− execute TCP.acceptOnPort basePort Array.defaultChunkSize handlerRW
110− let dataChunk = take Array.defaultChunkSize testDataSource
109+ execute TCP.acceptOnPort basePort defaultChunkSize handlerRW
110+ let dataChunk = take defaultChunkSize testDataSource
111111 Stream.eqBy (==) (Stream.fromList dataChunk) ls2
112112 assert res
113113
@@ -118,9 +118,9 @@ validateOnPortLocal = monadicIO $ do
118118 execute
119119 TCP.acceptOnPortLocal
120120 (basePort + 1)
121− Array.defaultChunkSize
121+ defaultChunkSize
122122 handlerRW
123− let dataChunk = take Array.defaultChunkSize testDataSource
123+ let dataChunk = take defaultChunkSize testDataSource
124124 Stream.eqBy (==) (Stream.fromList dataChunk) ls2
125125 assert res
126126
test/Streamly/Test/Network/Socket.hs+3−3
@@ -15,11 +15,11 @@ import Data.Function ((&))
1515import Data.Word (Word8)
1616import Network.Socket (Socket, PortNumber)
1717import Streamly.Internal.Control.Monad (discard)
18+import Streamly.Internal.System.IO (defaultChunkSize)
1819import Streamly.Prelude (SerialT)
1920import Test.QuickCheck (Property)
2021import Test.QuickCheck.Monadic (monadicIO, assert, run)
2122
22−import qualified Streamly.Internal.Data.Array.Foreign.Type as Array
2323import qualified Streamly.Internal.Network.Inet.TCP as TCP
2424import qualified Streamly.Internal.Network.Socket as Socket
2525import qualified Streamly.Internal.Unicode.Stream as Unicode
@@ -118,8 +118,8 @@ validateWithBufferOf = monadicIO $ do
118118validateRW :: Property
119119validateRW = monadicIO $ do
120120 res <- run $ do
121− ls2 <- execute (basePort + 1) Array.defaultChunkSize handlerRW
122− let dataChunk = take Array.defaultChunkSize testDataSource
121+ ls2 <- execute (basePort + 1) defaultChunkSize handlerRW
122+ let dataChunk = take defaultChunkSize testDataSource
123123 Stream.eqBy (==) (Stream.fromList dataChunk) ls2
124124 assert res
125125