4-- Module : Streamly.Internal.Network.Socket
5-- Copyright : (c) 2018 Composewell Technologies
8-- Maintainer : streamly@composewell.com
9-- Stability : experimental
12module Streamly.Internal.Network.Socket
19 -- * Accept connections
25 -- * Read from connection
38 , readChunksWithBufferOf
41 , toChunksWithBufferOf
45 -- * Write to connection
53 , fromBytesWithBufferOf
59 , writeChunksWithBufferOf
62 -- reading/writing datagrams
66import Control.Concurrent (threadWaitWrite, rtsSupportsBoundThreads)
67import Control.Exception (onException)
68import Control.Monad.Catch (MonadCatch, finally, MonadMask)
69import Control.Monad.IO.Class (MonadIO(..))
70import Control.Monad (forM_, when)
71import Data.Word (Word8)
72import Foreign.ForeignPtr (withForeignPtr)
73import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
74import Foreign.Ptr (minusPtr, plusPtr, Ptr, castPtr)
75import Foreign.Storable (Storable(..))
76import GHC.ForeignPtr (mallocPlainForeignPtrBytes)
78 (Socket, SocketOption(..), Family(..), SockAddr(..),
79 ProtocolNumber, withSocketsDo, SocketType(..), socket, bind,
80 setSocketOption, sendBuf, recvBuf)
81#if MIN_VERSION_network(3,1,0)
82import Network.Socket (withFdSocket)
84import Network.Socket (fdSocket)
86import Prelude hiding (read)
88import qualified Network.Socket as Net
90import Streamly.Prelude (MonadAsync)
91import Streamly.Internal.Data.Unfold.Types (Unfold(..))
92import Streamly.Internal.Data.Array.Stream.Foreign (lpackArraysChunksOf)
93import Streamly.Internal.Data.Array.Foreign.Types (Array(..))
94import Streamly.Internal.Data.Array.Foreign.Mut.Types (mutableArray)
95import Streamly.Internal.Data.Stream.Serial (SerialT)
96import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream)
97import Streamly.Data.Fold (Fold)
98-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
100import qualified Streamly.Data.Fold as FL
101import qualified Streamly.Internal.Data.Fold.Types as FL
102import qualified Streamly.Internal.Data.Unfold as UF
103import qualified Streamly.Internal.Data.Array.Foreign as IA
104import qualified Streamly.Data.Array.Foreign as A
105import qualified Streamly.Internal.Data.Array.Stream.Foreign as AS
106import qualified Streamly.Internal.Data.Array.Foreign.Types as A
107import qualified Streamly.Prelude as S
108import qualified Streamly.Internal.Data.Stream.StreamD.Type as D
110-- | @'handleWithM' socket act@ runs the monadic computation @act@ passing the
111-- socket handle to it. The handle will be closed on exit from 'handleWithM',
112-- whether by normal termination or by raising an exception. If closing the
113-- handle raises an exception, then this exception will be raised by
114-- 'handleWithM' rather than any exception raised by 'act'.
117{-# INLINE handleWithM #-}
118handleWithM :: (MonadMask m, MonadIO m) => (Socket -> m ()) -> Socket -> m ()
119handleWithM f sk = finally (f sk) (liftIO (Net.close sk))
121-- | Like 'handleWithM' but runs a streaming computation instead of a monadic
125{-# INLINE handleWith #-}
126handleWith :: (IsStream t, MonadAsync m, MonadCatch m)
127 => Socket -> (Socket -> t m a) -> t m a
128handleWith sk f = S.finally (liftIO $ Net.close sk) (f sk)
130-------------------------------------------------------------------------------
132-------------------------------------------------------------------------------
134-- XXX Protocol specific socket options should be separated from socket level
137-- | Specify the socket protocol details.
138data SockSpec = SockSpec
140 sockFamily :: !Family
141 , sockType :: !SocketType
142 , sockProto :: !ProtocolNumber
143 , sockOpts :: ![(SocketOption, Int)]
146initListener :: Int -> SockSpec -> SockAddr -> IO Socket
147initListener listenQLen SockSpec{..} addr =
149 sock <- socket sockFamily sockType sockProto
150 use sock `onException` Net.close sock
156 mapM_ (\(opt, val) -> setSocketOption sock opt val) sockOpts
158 Net.listen sock listenQLen
160{-# INLINE listenTuples #-}
161listenTuples :: MonadIO m
162 => Unfold m (Int, SockSpec, SockAddr) (Socket, SockAddr)
163listenTuples = Unfold step inject
165 inject (listenQLen, spec, addr) =
166 liftIO $ initListener listenQLen spec addr
169 r <- liftIO (Net.accept listener `onException` Net.close listener)
170 return $ D.Yield r listener
172-- | Unfold a three tuple @(listenQLen, spec, addr)@ into a stream of connected
173-- protocol sockets corresponding to incoming connections. @listenQLen@ is the
174-- maximum number of pending connections in the backlog. @spec@ is the socket
175-- protocol and options specification and @addr@ is the protocol address where
176-- the server listens for incoming connections.
180accept :: MonadIO m => Unfold m (Int, SockSpec, SockAddr) Socket
181accept = UF.map fst listenTuples
183{-# INLINE connectCommon #-}
184connectCommon :: SockSpec -> Maybe SockAddr -> SockAddr -> IO Socket
185connectCommon SockSpec{..} local remote = withSocketsDo $ do
186 sock <- socket sockFamily sockType sockProto
187 use sock `onException` Net.close sock
193 mapM_ (\(opt, val) -> setSocketOption sock opt val) sockOpts
194 forM_ local (bind sock)
195 Net.connect sock remote
197-- | Connect to a remote host using the given socket specification and remote
198-- address. Returns a connected socket or throws an exception.
202{-# INLINE connect #-}
203connect :: SockSpec -> SockAddr -> IO Socket
204connect spec = connectCommon spec Nothing
206-- | Connect to a remote host using the given socket specification, a local
207-- address to bind to and a remote address to connect to. Returns a connected
208-- socket or throws an exception.
212{-# INLINE connectFrom #-}
213connectFrom :: SockSpec -> SockAddr -> SockAddr -> IO Socket
214connectFrom spec local = connectCommon spec (Just local)
216-------------------------------------------------------------------------------
218-------------------------------------------------------------------------------
220{-# INLINE recvConnectionTuplesWith #-}
221recvConnectionTuplesWith :: MonadAsync m
222 => Int -> SockSpec -> SockAddr -> SerialT m (Socket, SockAddr)
223recvConnectionTuplesWith tcpListenQ spec addr = S.unfoldrM step Nothing
226 listener <- liftIO $ initListener tcpListenQ spec addr
227 r <- liftIO (Net.accept listener `onException` Net.close listener)
228 return $ Just (r, Just listener)
230 step (Just listener) = do
231 r <- liftIO (Net.accept listener `onException` Net.close listener)
232 return $ Just (r, Just listener)
234-- | Start a TCP stream server that listens for connections on the supplied
235-- server address specification (address family, local interface IP address and
236-- port). The server generates a stream of connected sockets. The first
237-- argument is the maximum number of pending connections in the backlog.
240{-# INLINE connections #-}
241connections :: MonadAsync m => Int -> SockSpec -> SockAddr -> SerialT m Socket
242connections tcpListenQ spec addr =
243 fst <$> recvConnectionTuplesWith tcpListenQ spec addr
245-------------------------------------------------------------------------------
247-------------------------------------------------------------------------------
249{-# INLINABLE readArrayUptoWith #-}
251 :: (h -> Ptr Word8 -> Int -> IO Int)
255readArrayUptoWith f size h = do
256 ptr <- mallocPlainForeignPtrBytes size
257 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
258 withForeignPtr ptr $ \p -> do
260 let v = A.unsafeFreeze
261 $ mutableArray ptr (p `plusPtr` n) (p `plusPtr` size)
263 -- XXX shrink only if the diff is significant
267-- | Read a 'ByteArray' from a file handle. If no data is available on the
268-- handle it blocks until some data becomes available. If data is available
269-- then it immediately returns that data without blocking. It reads a maximum
270-- of up to the size requested.
271{-# INLINABLE readArrayOf #-}
272readArrayOf :: Int -> Socket -> IO (Array Word8)
273readArrayOf = readArrayUptoWith recvBuf
275-------------------------------------------------------------------------------
277-------------------------------------------------------------------------------
279waitWhen0 :: Int -> Socket -> IO ()
280waitWhen0 0 s = when rtsSupportsBoundThreads $
281#if MIN_VERSION_network(3,1,0)
282 withFdSocket s $ \fd -> threadWaitWrite $ fromIntegral fd
283#elif MIN_VERSION_network(3,0,0)
284 fdSocket s >>= threadWaitWrite . fromIntegral
286 let fd = fdSocket s in threadWaitWrite $ fromIntegral fd
288waitWhen0 _ _ = return ()
290sendAll :: Socket -> Ptr Word8 -> Int -> IO ()
291sendAll _ _ len | len <= 0 = return ()
293 sent <- sendBuf s p len
295 -- assert (sent <= len)
296 when (sent >= 0) $ sendAll s (p `plusPtr` sent) (len - sent)
298{-# INLINABLE writeArrayWith #-}
299writeArrayWith :: Storable a
300 => (h -> Ptr Word8 -> Int -> IO ())
304writeArrayWith _ _ arr | A.length arr == 0 = return ()
305writeArrayWith f h Array{..} = withForeignPtr aStart $ \p ->
309 let p = unsafeForeignPtrToPtr aStart
312-- | Write an Array to a file handle.
315{-# INLINABLE writeChunk #-}
316writeChunk :: Storable a => Socket -> Array a -> IO ()
317writeChunk = writeArrayWith sendAll
319-------------------------------------------------------------------------------
320-- Stream of Arrays IO
321-------------------------------------------------------------------------------
323{-# INLINABLE _readChunksUptoWith #-}
324_readChunksUptoWith :: (IsStream t, MonadIO m)
325 => (Int -> h -> IO (Array Word8))
326 -> Int -> h -> t m (Array Word8)
327_readChunksUptoWith f size h = go
329 -- XXX use cons/nil instead
330 go = mkStream $ \_ yld _ stp -> do
331 arr <- liftIO $ f size h
336-- | @toChunksWithBufferOf size h@ reads a stream of arrays from file handle @h@.
337-- The maximum size of a single array is limited to @size@.
338-- 'fromHandleArraysUpto' ignores the prevailing 'TextEncoding' and 'NewlineMode'
340{-# INLINE_NORMAL toChunksWithBufferOf #-}
341toChunksWithBufferOf :: (IsStream t, MonadIO m)
342 => Int -> Socket -> t m (Array Word8)
343-- toChunksWithBufferOf = _readChunksUptoWith readArrayOf
344toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ())
346 {-# INLINE_LATE step #-}
348 arr <- liftIO $ readArrayOf size h
354-- XXX read 'Array a' instead of Word8
356-- | @toChunks h@ reads a stream of arrays from socket handle @h@.
357-- The maximum size of a single array is limited to @defaultChunkSize@.
360{-# INLINE toChunks #-}
361toChunks :: (IsStream t, MonadIO m) => Socket -> t m (Array Word8)
362toChunks = toChunksWithBufferOf A.defaultChunkSize
364-- | Unfold the tuple @(bufsize, socket)@ into a stream of 'Word8' arrays.
365-- Read requests to the socket are performed using a buffer of size @bufsize@.
366-- The size of an array in the resulting stream is always less than or equal to
370{-# INLINE_NORMAL readChunksWithBufferOf #-}
371readChunksWithBufferOf :: MonadIO m => Unfold m (Int, Socket) (Array Word8)
372readChunksWithBufferOf = Unfold step return
374 {-# INLINE_LATE step #-}
376 arr <- liftIO $ readArrayOf size h
380 _ -> D.Yield arr (size, h)
382-- | Unfolds a socket into a stream of 'Word8' arrays. Requests to the socket
383-- are performed using a buffer of size
384-- 'Streamly.Internal.Data.Array.Foreign.Types.defaultChunkSize'. The
385-- size of arrays in the resulting stream are therefore less than or equal to
386-- 'Streamly.Internal.Data.Array.Foreign.Types.defaultChunkSize'.
389{-# INLINE readChunks #-}
390readChunks :: MonadIO m => Unfold m Socket (Array Word8)
391readChunks = UF.supplyFirst readChunksWithBufferOf A.defaultChunkSize
393-------------------------------------------------------------------------------
394-- Read File to Stream
395-------------------------------------------------------------------------------
397-- TODO for concurrent streams implement readahead IO. We can send multiple
398-- read requests at the same time. For serial case we can use async IO. We can
399-- also control the read throughput in mbps or IOPS.
402-- | @readWithBufferOf bufsize handle@ reads a byte stream from a file
403-- handle, reads are performed in chunks of up to @bufsize@. The stream ends
404-- as soon as EOF is encountered.
406{-# INLINE readWithBufferOf #-}
407readWithBufferOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
408readWithBufferOf chunkSize h = A.flattenArrays $ readChunksUpto chunkSize h
412-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
414-- > read = 'readByChunks' A.defaultChunkSize
415-- | Generate a stream of elements of the given type from a socket. The
416-- stream ends when EOF is encountered.
419{-# INLINE toBytes #-}
420toBytes :: (IsStream t, MonadIO m) => Socket -> t m Word8
421toBytes = AS.concat . toChunks
423-- | Unfolds the tuple @(bufsize, socket)@ into a byte stream, read requests
424-- to the socket are performed using buffers of @bufsize@.
427{-# INLINE readWithBufferOf #-}
428readWithBufferOf :: MonadIO m => Unfold m (Int, Socket) Word8
429readWithBufferOf = UF.concat readChunksWithBufferOf A.read
431-- | Unfolds a 'Socket' into a byte stream. IO requests to the socket are
432-- performed in sizes of
433-- 'Streamly.Internal.Data.Array.Foreign.Types.defaultChunkSize'.
437read :: MonadIO m => Unfold m Socket Word8
438read = UF.supplyFirst readWithBufferOf A.defaultChunkSize
440-------------------------------------------------------------------------------
442-------------------------------------------------------------------------------
444-- | Write a stream of arrays to a handle.
447{-# INLINE fromChunks #-}
448fromChunks :: (MonadIO m, Storable a)
449 => Socket -> SerialT m (Array a) -> m ()
450fromChunks h = S.mapM_ (liftIO . writeChunk h)
452-- | Write a stream of arrays to a socket. Each array in the stream is written
453-- to the socket as a separate IO request.
456{-# INLINE writeChunks #-}
457writeChunks :: (MonadIO m, Storable a) => Socket -> Fold m (Array a) ()
458writeChunks h = FL.drainBy (liftIO . writeChunk h)
460-- | @writeChunksWithBufferOf bufsize socket@ writes a stream of arrays
461-- to @socket@ after coalescing the adjacent arrays in chunks of @bufsize@.
462-- We never split an array, if a single array is bigger than the specified size
463-- it emitted as it is. Multiple arrays are coalesed as long as the total size
464-- remains below the specified size.
467{-# INLINE writeChunksWithBufferOf #-}
468writeChunksWithBufferOf :: (MonadIO m, Storable a)
469 => Int -> Socket -> Fold m (Array a) ()
470writeChunksWithBufferOf n h = lpackArraysChunksOf n (writeChunks h)
473-- | Write a stream of strings to a socket in Latin1 encoding. Output is
474-- flushed to the socket for each string.
478{-# INLINE writeStrings #-}
479writeStrings :: MonadIO m
480 => (SerialT m Char -> SerialT m Word8) -> Socket -> Fold m String ()
481writeStrings encode h =
482 FL.lmapM (IA.fromStream . encode . S.fromList) (writeChunks h)
484-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
487-- Note that if you use a chunk size less than 8K (GHC's default buffer
488-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
489-- do not want buffering to occur at GHC level as well. Same thing applies to
492-- | Like 'write' but provides control over the write buffer. Output will
493-- be written to the IO device as soon as we collect the specified number of
497{-# INLINE fromBytesWithBufferOf #-}
498fromBytesWithBufferOf :: MonadIO m => Int -> Socket -> SerialT m Word8 -> m ()
499fromBytesWithBufferOf n h m = fromChunks h $ AS.arraysOf n m
501-- | Write a byte stream to a socket. Accumulates the input in chunks of
502-- specified number of bytes before writing.
505{-# INLINE writeWithBufferOf #-}
506writeWithBufferOf :: MonadIO m => Int -> Socket -> Fold m Word8 ()
507writeWithBufferOf n h = FL.chunksOf n (A.writeNUnsafe n) (writeChunks h)
509-- > write = 'writeWithBufferOf' A.defaultChunkSize
511-- | Write a byte stream to a file handle. Combines the bytes in chunks of size
512-- up to 'A.defaultChunkSize' before writing. Note that the write behavior
513-- depends on the 'IOMode' and the current seek position of the handle.
516{-# INLINE fromBytes #-}
517fromBytes :: MonadIO m => Socket -> SerialT m Word8 -> m ()
518fromBytes = fromBytesWithBufferOf A.defaultChunkSize
520-- | Write a byte stream to a socket. Accumulates the input in chunks of
521-- up to 'A.defaultChunkSize' bytes before writing.
524-- write = 'writeWithBufferOf' 'A.defaultChunkSize'
529write :: MonadIO m => Socket -> Fold m Word8 ()
530write = writeWithBufferOf A.defaultChunkSize
534write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
535write = toHandleWith A.defaultChunkSize
538-------------------------------------------------------------------------------
539-- IO with encoding/decoding Unicode characters
540-------------------------------------------------------------------------------
544-- > readUtf8 = decodeUtf8 . read
546-- Read a UTF8 encoded stream of unicode characters from a file handle.
549{-# INLINE readUtf8 #-}
550readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
551readUtf8 = decodeUtf8 . read
554-- > writeUtf8 h s = write h $ encodeUtf8 s
556-- Encode a stream of unicode characters to UTF8 and write it to the given file
557-- handle. Default block buffering applies to the writes.
560{-# INLINE writeUtf8 #-}
561writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
562writeUtf8 h s = write h $ encodeUtf8 s
564-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
565-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
566-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
567-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
571{-# INLINE writeUtf8ByLines #-}
572writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
573writeUtf8ByLines = undefined
575-- | Read UTF-8 lines from a file handle and apply the specified fold to each
576-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
579{-# INLINE readLines #-}
580readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
581readLines h f = foldLines (readUtf8 h) f
583-------------------------------------------------------------------------------
584-- Framing on a sequence
585-------------------------------------------------------------------------------
587-- | Read a stream from a file handle and split it into frames delimited by
588-- the specified sequence of elements. The supplied fold is applied on each
592{-# INLINE readFrames #-}
593readFrames :: (IsStream t, MonadIO m, Storable a)
594 => Array a -> Handle -> Fold m a b -> t m b
595readFrames = undefined -- foldFrames . read
597-- | Write a stream to the given file handle buffering up to frames separated
598-- by the given sequence or up to a maximum of @defaultChunkSize@.
601{-# INLINE writeByFrames #-}
602writeByFrames :: (IsStream t, MonadIO m, Storable a)
603 => Array a -> Handle -> t m a -> m ()
604writeByFrames = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list