1{-# OPTIONS_HADDOCK hide #-}
3{-# LANGUAGE BangPatterns #-}
4{-# LANGUAGE FlexibleContexts #-}
5{-# LANGUAGE MagicHash #-}
6{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE UnboxedTuples #-}
12-- Module : Streamly.Internal.Network.Socket
13-- Copyright : (c) 2018 Composewell Technologies
16-- Maintainer : streamly@composewell.com
17-- Stability : experimental
20module Streamly.Internal.Network.Socket
27 -- * Accept connections
31 -- * Read from connection
44 , readChunksWithBufferOf
47 , toChunksWithBufferOf
51 -- * Write to connection
59 , fromBytesWithBufferOf
65 , writeChunksWithBufferOf
68 -- reading/writing datagrams
72import Control.Concurrent (threadWaitWrite, rtsSupportsBoundThreads)
73import Control.Monad.Catch (MonadCatch, finally, MonadMask)
74import Control.Monad.IO.Class (MonadIO(..))
75import Control.Monad (when)
76import Data.Word (Word8)
77import Foreign.ForeignPtr (withForeignPtr)
78import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
79import Foreign.Ptr (minusPtr, plusPtr, Ptr, castPtr)
80import Foreign.Storable (Storable(..))
81import GHC.ForeignPtr (mallocPlainForeignPtrBytes)
82import Network.Socket (sendBuf, recvBuf)
84 (Socket, SocketOption(..), Family(..), SockAddr(..),
85 ProtocolNumber, withSocketsDo, SocketType(..), socket, bind,
87#if MIN_VERSION_network(3,1,0)
88import Network.Socket (withFdSocket)
90import Network.Socket (fdSocket)
92import Prelude hiding (read)
94import qualified Network.Socket as Net
96import Streamly (MonadAsync)
97import Streamly.Internal.Data.Unfold.Types (Unfold(..))
98import Streamly.Internal.Memory.Array.Types (Array(..), lpackArraysChunksOf)
99import Streamly.Internal.Data.Stream.Serial (SerialT)
100import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream)
101import Streamly.Data.Fold (Fold)
102-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
104import qualified Streamly.Data.Fold as FL
105import qualified Streamly.Internal.Data.Fold.Types as FL
106import qualified Streamly.Internal.Data.Unfold as UF
107import qualified Streamly.Internal.Memory.Array as IA
108import qualified Streamly.Memory.Array as A
109import qualified Streamly.Internal.Memory.ArrayStream as AS
110import qualified Streamly.Internal.Memory.Array.Types as A
111import qualified Streamly.Prelude as S
112import qualified Streamly.Internal.Data.Stream.StreamD.Type as D
114-- | @'handleWithM' socket act@ runs the monadic computation @act@ passing the
115-- socket handle to it. The handle will be closed on exit from 'handleWithM',
116-- whether by normal termination or by raising an exception. If closing the
117-- handle raises an exception, then this exception will be raised by
118-- 'handleWithM' rather than any exception raised by 'act'.
121{-# INLINE handleWithM #-}
122handleWithM :: (MonadMask m, MonadIO m) => (Socket -> m ()) -> Socket -> m ()
123handleWithM f sk = finally (f sk) (liftIO (Net.close sk))
125-- | Like 'handleWithM' but runs a streaming computation instead of a monadic
129{-# INLINE handleWith #-}
130handleWith :: (IsStream t, MonadCatch m, MonadIO m)
131 => Socket -> (Socket -> t m a) -> t m a
132handleWith sk f = S.finally (liftIO $ Net.close sk) (f sk)
134-------------------------------------------------------------------------------
136-------------------------------------------------------------------------------
138-- XXX Protocol specific socket options should be separated from socket level
141-- | Specify the socket protocol details.
142data SockSpec = SockSpec
144 sockFamily :: !Family
145 , sockType :: !SocketType
146 , sockProto :: !ProtocolNumber
147 , sockOpts :: ![(SocketOption, Int)]
150initListener :: Int -> SockSpec -> SockAddr -> IO Socket
151initListener listenQLen SockSpec{..} addr =
153 sock <- socket sockFamily sockType sockProto
154 mapM_ (\(opt, val) -> setSocketOption sock opt val) sockOpts
156 Net.listen sock listenQLen
159{-# INLINE listenTuples #-}
160listenTuples :: MonadIO m
161 => Unfold m (Int, SockSpec, SockAddr) (Socket, SockAddr)
162listenTuples = Unfold step inject
164 inject (listenQLen, spec, addr) = do
165 listener <- liftIO $ initListener listenQLen spec addr
169 r <- liftIO $ Net.accept listener
170 -- XXX error handling
171 return $ D.Yield r listener
173-- | Unfold a three tuple @(listenQLen, spec, addr)@ into a stream of connected
174-- protocol sockets corresponding to incoming connections. @listenQLen@ is the
175-- maximum number of pending connections in the backlog. @spec@ is the socket
176-- protocol and options specification and @addr@ is the protocol address where
177-- the server listens for incoming connections.
181accept :: MonadIO m => Unfold m (Int, SockSpec, SockAddr) Socket
182accept = UF.map fst listenTuples
184-------------------------------------------------------------------------------
186-------------------------------------------------------------------------------
188{-# INLINE recvConnectionTuplesWith #-}
189recvConnectionTuplesWith :: MonadAsync m
190 => Int -> SockSpec -> SockAddr -> SerialT m (Socket, SockAddr)
191recvConnectionTuplesWith tcpListenQ spec addr = S.unfoldrM step Nothing
194 listener <- liftIO $ initListener tcpListenQ spec addr
195 r <- liftIO $ Net.accept listener
196 -- XXX error handling
197 return $ Just (r, Just listener)
199 step (Just listener) = do
200 r <- liftIO $ Net.accept listener
201 -- XXX error handling
202 return $ Just (r, Just listener)
204-- | Start a TCP stream server that listens for connections on the supplied
205-- server address specification (address family, local interface IP address and
206-- port). The server generates a stream of connected sockets. The first
207-- argument is the maximum number of pending connections in the backlog.
210{-# INLINE connections #-}
211connections :: MonadAsync m => Int -> SockSpec -> SockAddr -> SerialT m Socket
212connections tcpListenQ spec addr = fmap fst $
213 recvConnectionTuplesWith tcpListenQ spec addr
215-------------------------------------------------------------------------------
217-------------------------------------------------------------------------------
219{-# INLINABLE readArrayUptoWith #-}
221 :: (h -> Ptr Word8 -> Int -> IO Int)
225readArrayUptoWith f size h = do
226 ptr <- mallocPlainForeignPtrBytes size
227 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
228 withForeignPtr ptr $ \p -> do
232 , aEnd = p `plusPtr` n
233 , aBound = p `plusPtr` size
235 -- XXX shrink only if the diff is significant
239-- | Read a 'ByteArray' from a file handle. If no data is available on the
240-- handle it blocks until some data becomes available. If data is available
241-- then it immediately returns that data without blocking. It reads a maximum
242-- of up to the size requested.
243{-# INLINABLE readArrayOf #-}
244readArrayOf :: Int -> Socket -> IO (Array Word8)
245readArrayOf = readArrayUptoWith recvBuf
247-------------------------------------------------------------------------------
249-------------------------------------------------------------------------------
251waitWhen0 :: Int -> Socket -> IO ()
252waitWhen0 0 s = when rtsSupportsBoundThreads $
253#if MIN_VERSION_network(3,1,0)
254 withFdSocket s $ \fd -> threadWaitWrite $ fromIntegral fd
256 let fd = fdSocket s in threadWaitWrite $ fromIntegral fd
258waitWhen0 _ _ = return ()
260sendAll :: Socket -> Ptr Word8 -> Int -> IO ()
261sendAll _ _ len | len <= 0 = return ()
263 sent <- sendBuf s p len
265 -- assert (sent <= len)
266 when (sent >= 0) $ sendAll s (p `plusPtr` sent) (len - sent)
268{-# INLINABLE writeArrayWith #-}
269writeArrayWith :: Storable a
270 => (h -> Ptr Word8 -> Int -> IO ())
274writeArrayWith _ _ arr | A.length arr == 0 = return ()
275writeArrayWith f h Array{..} = withForeignPtr aStart $ \p ->
279 let p = unsafeForeignPtrToPtr aStart
282-- | Write an Array to a file handle.
285{-# INLINABLE writeChunk #-}
286writeChunk :: Storable a => Socket -> Array a -> IO ()
287writeChunk = writeArrayWith sendAll
289-------------------------------------------------------------------------------
290-- Stream of Arrays IO
291-------------------------------------------------------------------------------
293{-# INLINABLE _readChunksUptoWith #-}
294_readChunksUptoWith :: (IsStream t, MonadIO m)
295 => (Int -> h -> IO (Array Word8))
296 -> Int -> h -> t m (Array Word8)
297_readChunksUptoWith f size h = go
299 -- XXX use cons/nil instead
300 go = mkStream $ \_ yld _ stp -> do
301 arr <- liftIO $ f size h
306-- | @toChunksWithBufferOf size h@ reads a stream of arrays from file handle @h@.
307-- The maximum size of a single array is limited to @size@.
308-- 'fromHandleArraysUpto' ignores the prevailing 'TextEncoding' and 'NewlineMode'
310{-# INLINE_NORMAL toChunksWithBufferOf #-}
311toChunksWithBufferOf :: (IsStream t, MonadIO m)
312 => Int -> Socket -> t m (Array Word8)
313-- toChunksWithBufferOf = _readChunksUptoWith readArrayOf
314toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ())
316 {-# INLINE_LATE step #-}
318 arr <- liftIO $ readArrayOf size h
324-- XXX read 'Array a' instead of Word8
326-- | @toChunks h@ reads a stream of arrays from socket handle @h@.
327-- The maximum size of a single array is limited to @defaultChunkSize@.
330{-# INLINE toChunks #-}
331toChunks :: (IsStream t, MonadIO m) => Socket -> t m (Array Word8)
332toChunks = toChunksWithBufferOf A.defaultChunkSize
334-- | Unfold the tuple @(bufsize, socket)@ into a stream of 'Word8' arrays.
335-- Read requests to the socket are performed using a buffer of size @bufsize@.
336-- The size of an array in the resulting stream is always less than or equal to
340{-# INLINE_NORMAL readChunksWithBufferOf #-}
341readChunksWithBufferOf :: MonadIO m => Unfold m (Int, Socket) (Array Word8)
342readChunksWithBufferOf = Unfold step return
344 {-# INLINE_LATE step #-}
346 arr <- liftIO $ readArrayOf size h
350 _ -> D.Yield arr (size, h)
352-- | Unfolds a socket into a stream of 'Word8' arrays. Requests to the socket
353-- are performed using a buffer of size
354-- 'Streamly.Internal.Memory.Array.Types.defaultChunkSize'. The
355-- size of arrays in the resulting stream are therefore less than or equal to
356-- 'Streamly.Internal.Memory.Array.Types.defaultChunkSize'.
359{-# INLINE readChunks #-}
360readChunks :: MonadIO m => Unfold m Socket (Array Word8)
361readChunks = UF.supplyFirst readChunksWithBufferOf A.defaultChunkSize
363-------------------------------------------------------------------------------
364-- Read File to Stream
365-------------------------------------------------------------------------------
367-- TODO for concurrent streams implement readahead IO. We can send multiple
368-- read requests at the same time. For serial case we can use async IO. We can
369-- also control the read throughput in mbps or IOPS.
372-- | @readWithBufferOf bufsize handle@ reads a byte stream from a file
373-- handle, reads are performed in chunks of up to @bufsize@. The stream ends
374-- as soon as EOF is encountered.
376{-# INLINE readWithBufferOf #-}
377readWithBufferOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
378readWithBufferOf chunkSize h = A.flattenArrays $ readChunksUpto chunkSize h
382-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
384-- > read = 'readByChunks' A.defaultChunkSize
385-- | Generate a stream of elements of the given type from a socket. The
386-- stream ends when EOF is encountered.
389{-# INLINE toBytes #-}
390toBytes :: (IsStream t, MonadIO m) => Socket -> t m Word8
391toBytes = AS.concat . toChunks
393-- | Unfolds the tuple @(bufsize, socket)@ into a byte stream, read requests
394-- to the socket are performed using buffers of @bufsize@.
397{-# INLINE readWithBufferOf #-}
398readWithBufferOf :: MonadIO m => Unfold m (Int, Socket) Word8
399readWithBufferOf = UF.concat readChunksWithBufferOf A.read
401-- | Unfolds a 'Socket' into a byte stream. IO requests to the socket are
402-- performed in sizes of
403-- 'Streamly.Internal.Memory.Array.Types.defaultChunkSize'.
407read :: MonadIO m => Unfold m Socket Word8
408read = UF.supplyFirst readWithBufferOf A.defaultChunkSize
410-------------------------------------------------------------------------------
412-------------------------------------------------------------------------------
414-- | Write a stream of arrays to a handle.
417{-# INLINE fromChunks #-}
418fromChunks :: (MonadIO m, Storable a)
419 => Socket -> SerialT m (Array a) -> m ()
420fromChunks h m = S.mapM_ (liftIO . writeChunk h) m
422-- | Write a stream of arrays to a socket. Each array in the stream is written
423-- to the socket as a separate IO request.
426{-# INLINE writeChunks #-}
427writeChunks :: (MonadIO m, Storable a) => Socket -> Fold m (Array a) ()
428writeChunks h = FL.drainBy (liftIO . writeChunk h)
430-- | @writeChunksWithBufferOf bufsize socket@ writes a stream of arrays
431-- to @socket@ after coalescing the adjacent arrays in chunks of @bufsize@.
432-- We never split an array, if a single array is bigger than the specified size
433-- it emitted as it is. Multiple arrays are coalesed as long as the total size
434-- remains below the specified size.
437{-# INLINE writeChunksWithBufferOf #-}
438writeChunksWithBufferOf :: (MonadIO m, Storable a)
439 => Int -> Socket -> Fold m (Array a) ()
440writeChunksWithBufferOf n h = lpackArraysChunksOf n (writeChunks h)
442-- | Write a stream of strings to a socket in Latin1 encoding. Output is
443-- flushed to the socket for each string.
447{-# INLINE writeStrings #-}
448writeStrings :: MonadIO m
449 => (SerialT m Char -> SerialT m Word8) -> Socket -> Fold m String ()
450writeStrings encode h =
451 FL.lmapM (IA.fromStream . encode . S.fromList) (writeChunks h)
453-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
456-- Note that if you use a chunk size less than 8K (GHC's default buffer
457-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
458-- do not want buffering to occur at GHC level as well. Same thing applies to
461-- | Like 'write' but provides control over the write buffer. Output will
462-- be written to the IO device as soon as we collect the specified number of
466{-# INLINE fromBytesWithBufferOf #-}
467fromBytesWithBufferOf :: MonadIO m => Int -> Socket -> SerialT m Word8 -> m ()
468fromBytesWithBufferOf n h m = fromChunks h $ AS.arraysOf n m
470-- | Write a byte stream to a socket. Accumulates the input in chunks of
471-- specified number of bytes before writing.
474{-# INLINE writeWithBufferOf #-}
475writeWithBufferOf :: MonadIO m => Int -> Socket -> Fold m Word8 ()
476writeWithBufferOf n h = FL.lchunksOf n (A.writeNUnsafe n) (writeChunks h)
478-- > write = 'writeWithBufferOf' A.defaultChunkSize
480-- | Write a byte stream to a file handle. Combines the bytes in chunks of size
481-- up to 'A.defaultChunkSize' before writing. Note that the write behavior
482-- depends on the 'IOMode' and the current seek position of the handle.
485{-# INLINE fromBytes #-}
486fromBytes :: MonadIO m => Socket -> SerialT m Word8 -> m ()
487fromBytes = fromBytesWithBufferOf A.defaultChunkSize
489-- | Write a byte stream to a socket. Accumulates the input in chunks of
490-- up to 'A.defaultChunkSize' bytes before writing.
493-- write = 'writeWithBufferOf' 'A.defaultChunkSize'
498write :: MonadIO m => Socket -> Fold m Word8 ()
499write = writeWithBufferOf A.defaultChunkSize
503write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
504write = toHandleWith A.defaultChunkSize
507-------------------------------------------------------------------------------
508-- IO with encoding/decoding Unicode characters
509-------------------------------------------------------------------------------
513-- > readUtf8 = decodeUtf8 . read
515-- Read a UTF8 encoded stream of unicode characters from a file handle.
518{-# INLINE readUtf8 #-}
519readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
520readUtf8 = decodeUtf8 . read
523-- > writeUtf8 h s = write h $ encodeUtf8 s
525-- Encode a stream of unicode characters to UTF8 and write it to the given file
526-- handle. Default block buffering applies to the writes.
529{-# INLINE writeUtf8 #-}
530writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
531writeUtf8 h s = write h $ encodeUtf8 s
533-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
534-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
535-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
536-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
540{-# INLINE writeUtf8ByLines #-}
541writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
542writeUtf8ByLines = undefined
544-- | Read UTF-8 lines from a file handle and apply the specified fold to each
545-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
548{-# INLINE readLines #-}
549readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
550readLines h f = foldLines (readUtf8 h) f
552-------------------------------------------------------------------------------
553-- Framing on a sequence
554-------------------------------------------------------------------------------
556-- | Read a stream from a file handle and split it into frames delimited by
557-- the specified sequence of elements. The supplied fold is applied on each
561{-# INLINE readFrames #-}
562readFrames :: (IsStream t, MonadIO m, Storable a)
563 => Array a -> Handle -> Fold m a b -> t m b
564readFrames = undefined -- foldFrames . read
566-- | Write a stream to the given file handle buffering up to frames separated
567-- by the given sequence or up to a maximum of @defaultChunkSize@.
570{-# INLINE writeByFrames #-}
571writeByFrames :: (IsStream t, MonadIO m, Storable a)
572 => Array a -> Handle -> t m a -> m ()
573writeByFrames = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list