2{-# LANGUAGE FlexibleContexts #-}
3{-# LANGUAGE RecordWildCards #-}
8-- Module : Streamly.Internal.Network.Socket
9-- Copyright : (c) 2018 Composewell Technologies
12-- Maintainer : streamly@composewell.com
13-- Stability : experimental
16module Streamly.Internal.Network.Socket
23 -- * Accept connections
29 -- * Read from connection
42 , readChunksWithBufferOf
45 , toChunksWithBufferOf
49 -- * Write to connection
57 , fromBytesWithBufferOf
63 , writeChunksWithBufferOf
66 -- reading/writing datagrams
70import Control.Concurrent (threadWaitWrite, rtsSupportsBoundThreads)
71import Control.Exception (onException)
72import Control.Monad.Catch (MonadCatch, finally, MonadMask)
73import Control.Monad.IO.Class (MonadIO(..))
74import Control.Monad (when)
75import Data.Word (Word8)
76import Foreign.ForeignPtr (withForeignPtr)
77import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
78import Foreign.Ptr (minusPtr, plusPtr, Ptr, castPtr)
79import Foreign.Storable (Storable(..))
80import GHC.ForeignPtr (mallocPlainForeignPtrBytes)
82 (Socket, SocketOption(..), Family(..), SockAddr(..),
83 ProtocolNumber, withSocketsDo, SocketType(..), socket, bind,
84 setSocketOption, sendBuf, recvBuf)
85#if MIN_VERSION_network(3,1,0)
86import Network.Socket (withFdSocket)
88import Network.Socket (fdSocket)
90import Prelude hiding (read)
92import qualified Network.Socket as Net
94import Streamly (MonadAsync)
95import Streamly.Internal.Data.Unfold.Types (Unfold(..))
96import Streamly.Internal.Memory.Array.Types (Array(..), lpackArraysChunksOf)
97import Streamly.Internal.Data.Stream.Serial (SerialT)
98import Streamly.Internal.Data.Stream.StreamK.Type (IsStream, mkStream)
99import Streamly.Data.Fold (Fold)
100-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
102import qualified Streamly.Data.Fold as FL
103import qualified Streamly.Internal.Data.Fold.Types as FL
104import qualified Streamly.Internal.Data.Unfold as UF
105import qualified Streamly.Internal.Memory.Array as IA
106import qualified Streamly.Memory.Array as A
107import qualified Streamly.Internal.Memory.ArrayStream as AS
108import qualified Streamly.Internal.Memory.Array.Types as A
109import qualified Streamly.Prelude as S
110import qualified Streamly.Internal.Data.Stream.StreamD.Type as D
112-- | @'handleWithM' socket act@ runs the monadic computation @act@ passing the
113-- socket handle to it. The handle will be closed on exit from 'handleWithM',
114-- whether by normal termination or by raising an exception. If closing the
115-- handle raises an exception, then this exception will be raised by
116-- 'handleWithM' rather than any exception raised by 'act'.
119{-# INLINE handleWithM #-}
120handleWithM :: (MonadMask m, MonadIO m) => (Socket -> m ()) -> Socket -> m ()
121handleWithM f sk = finally (f sk) (liftIO (Net.close sk))
123-- | Like 'handleWithM' but runs a streaming computation instead of a monadic
127{-# INLINE handleWith #-}
128handleWith :: (IsStream t, MonadCatch m, MonadIO m)
129 => Socket -> (Socket -> t m a) -> t m a
130handleWith sk f = S.finally (liftIO $ Net.close sk) (f sk)
132-------------------------------------------------------------------------------
134-------------------------------------------------------------------------------
136-- XXX Protocol specific socket options should be separated from socket level
139-- | Specify the socket protocol details.
140data SockSpec = SockSpec
142 sockFamily :: !Family
143 , sockType :: !SocketType
144 , sockProto :: !ProtocolNumber
145 , sockOpts :: ![(SocketOption, Int)]
148initListener :: Int -> SockSpec -> SockAddr -> IO Socket
149initListener listenQLen SockSpec{..} addr =
151 sock <- socket sockFamily sockType sockProto
152 use sock `onException` Net.close sock
158 mapM_ (\(opt, val) -> setSocketOption sock opt val) sockOpts
160 Net.listen sock listenQLen
162{-# INLINE listenTuples #-}
163listenTuples :: MonadIO m
164 => Unfold m (Int, SockSpec, SockAddr) (Socket, SockAddr)
165listenTuples = Unfold step inject
167 inject (listenQLen, spec, addr) =
168 liftIO $ initListener listenQLen spec addr
171 r <- liftIO $ (Net.accept listener `onException` Net.close listener)
172 return $ D.Yield r listener
174-- | Unfold a three tuple @(listenQLen, spec, addr)@ into a stream of connected
175-- protocol sockets corresponding to incoming connections. @listenQLen@ is the
176-- maximum number of pending connections in the backlog. @spec@ is the socket
177-- protocol and options specification and @addr@ is the protocol address where
178-- the server listens for incoming connections.
182accept :: MonadIO m => Unfold m (Int, SockSpec, SockAddr) Socket
183accept = UF.map fst listenTuples
185{-# INLINE connectCommon #-}
186connectCommon :: SockSpec -> Maybe SockAddr -> SockAddr -> IO Socket
187connectCommon SockSpec{..} local remote = withSocketsDo $ do
188 sock <- socket sockFamily sockType sockProto
189 use sock `onException` Net.close sock
195 mapM_ (\(opt, val) -> setSocketOption sock opt val) sockOpts
198 Just addr -> bind sock addr
199 Net.connect sock remote
201-- | Connect to a remote host using the given socket specification and remote
202-- address. Returns a connected socket or throws an exception.
206{-# INLINE connect #-}
207connect :: SockSpec -> SockAddr -> IO Socket
208connect spec remote = connectCommon spec Nothing remote
210-- | Connect to a remote host using the given socket specification, a local
211-- address to bind to and a remote address to connect to. Returns a connected
212-- socket or throws an exception.
216{-# INLINE connectFrom #-}
217connectFrom :: SockSpec -> SockAddr -> SockAddr -> IO Socket
218connectFrom spec local remote = connectCommon spec (Just local) remote
220-------------------------------------------------------------------------------
222-------------------------------------------------------------------------------
224{-# INLINE recvConnectionTuplesWith #-}
225recvConnectionTuplesWith :: MonadAsync m
226 => Int -> SockSpec -> SockAddr -> SerialT m (Socket, SockAddr)
227recvConnectionTuplesWith tcpListenQ spec addr = S.unfoldrM step Nothing
230 listener <- liftIO $ initListener tcpListenQ spec addr
231 r <- liftIO $ (Net.accept listener `onException` Net.close listener)
232 return $ Just (r, Just listener)
234 step (Just listener) = do
235 r <- liftIO $ (Net.accept listener `onException` Net.close listener)
236 return $ Just (r, Just listener)
238-- | Start a TCP stream server that listens for connections on the supplied
239-- server address specification (address family, local interface IP address and
240-- port). The server generates a stream of connected sockets. The first
241-- argument is the maximum number of pending connections in the backlog.
244{-# INLINE connections #-}
245connections :: MonadAsync m => Int -> SockSpec -> SockAddr -> SerialT m Socket
246connections tcpListenQ spec addr =
247 fst <$> recvConnectionTuplesWith tcpListenQ spec addr
249-------------------------------------------------------------------------------
251-------------------------------------------------------------------------------
253{-# INLINABLE readArrayUptoWith #-}
255 :: (h -> Ptr Word8 -> Int -> IO Int)
259readArrayUptoWith f size h = do
260 ptr <- mallocPlainForeignPtrBytes size
261 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
262 withForeignPtr ptr $ \p -> do
266 , aEnd = p `plusPtr` n
267 , aBound = p `plusPtr` size
269 -- XXX shrink only if the diff is significant
273-- | Read a 'ByteArray' from a file handle. If no data is available on the
274-- handle it blocks until some data becomes available. If data is available
275-- then it immediately returns that data without blocking. It reads a maximum
276-- of up to the size requested.
277{-# INLINABLE readArrayOf #-}
278readArrayOf :: Int -> Socket -> IO (Array Word8)
279readArrayOf = readArrayUptoWith recvBuf
281-------------------------------------------------------------------------------
283-------------------------------------------------------------------------------
285waitWhen0 :: Int -> Socket -> IO ()
286waitWhen0 0 s = when rtsSupportsBoundThreads $
287#if MIN_VERSION_network(3,1,0)
288 withFdSocket s $ \fd -> threadWaitWrite $ fromIntegral fd
289#elif MIN_VERSION_network(3,0,0)
290 fdSocket s >>= threadWaitWrite . fromIntegral
292 let fd = fdSocket s in threadWaitWrite $ fromIntegral fd
294waitWhen0 _ _ = return ()
296sendAll :: Socket -> Ptr Word8 -> Int -> IO ()
297sendAll _ _ len | len <= 0 = return ()
299 sent <- sendBuf s p len
301 -- assert (sent <= len)
302 when (sent >= 0) $ sendAll s (p `plusPtr` sent) (len - sent)
304{-# INLINABLE writeArrayWith #-}
305writeArrayWith :: Storable a
306 => (h -> Ptr Word8 -> Int -> IO ())
310writeArrayWith _ _ arr | A.length arr == 0 = return ()
311writeArrayWith f h Array{..} = withForeignPtr aStart $ \p ->
315 let p = unsafeForeignPtrToPtr aStart
318-- | Write an Array to a file handle.
321{-# INLINABLE writeChunk #-}
322writeChunk :: Storable a => Socket -> Array a -> IO ()
323writeChunk = writeArrayWith sendAll
325-------------------------------------------------------------------------------
326-- Stream of Arrays IO
327-------------------------------------------------------------------------------
329{-# INLINABLE _readChunksUptoWith #-}
330_readChunksUptoWith :: (IsStream t, MonadIO m)
331 => (Int -> h -> IO (Array Word8))
332 -> Int -> h -> t m (Array Word8)
333_readChunksUptoWith f size h = go
335 -- XXX use cons/nil instead
336 go = mkStream $ \_ yld _ stp -> do
337 arr <- liftIO $ f size h
342-- | @toChunksWithBufferOf size h@ reads a stream of arrays from file handle @h@.
343-- The maximum size of a single array is limited to @size@.
344-- 'fromHandleArraysUpto' ignores the prevailing 'TextEncoding' and 'NewlineMode'
346{-# INLINE_NORMAL toChunksWithBufferOf #-}
347toChunksWithBufferOf :: (IsStream t, MonadIO m)
348 => Int -> Socket -> t m (Array Word8)
349-- toChunksWithBufferOf = _readChunksUptoWith readArrayOf
350toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ())
352 {-# INLINE_LATE step #-}
354 arr <- liftIO $ readArrayOf size h
360-- XXX read 'Array a' instead of Word8
362-- | @toChunks h@ reads a stream of arrays from socket handle @h@.
363-- The maximum size of a single array is limited to @defaultChunkSize@.
366{-# INLINE toChunks #-}
367toChunks :: (IsStream t, MonadIO m) => Socket -> t m (Array Word8)
368toChunks = toChunksWithBufferOf A.defaultChunkSize
370-- | Unfold the tuple @(bufsize, socket)@ into a stream of 'Word8' arrays.
371-- Read requests to the socket are performed using a buffer of size @bufsize@.
372-- The size of an array in the resulting stream is always less than or equal to
376{-# INLINE_NORMAL readChunksWithBufferOf #-}
377readChunksWithBufferOf :: MonadIO m => Unfold m (Int, Socket) (Array Word8)
378readChunksWithBufferOf = Unfold step return
380 {-# INLINE_LATE step #-}
382 arr <- liftIO $ readArrayOf size h
386 _ -> D.Yield arr (size, h)
388-- | Unfolds a socket into a stream of 'Word8' arrays. Requests to the socket
389-- are performed using a buffer of size
390-- 'Streamly.Internal.Memory.Array.Types.defaultChunkSize'. The
391-- size of arrays in the resulting stream are therefore less than or equal to
392-- 'Streamly.Internal.Memory.Array.Types.defaultChunkSize'.
395{-# INLINE readChunks #-}
396readChunks :: MonadIO m => Unfold m Socket (Array Word8)
397readChunks = UF.supplyFirst readChunksWithBufferOf A.defaultChunkSize
399-------------------------------------------------------------------------------
400-- Read File to Stream
401-------------------------------------------------------------------------------
403-- TODO for concurrent streams implement readahead IO. We can send multiple
404-- read requests at the same time. For serial case we can use async IO. We can
405-- also control the read throughput in mbps or IOPS.
408-- | @readWithBufferOf bufsize handle@ reads a byte stream from a file
409-- handle, reads are performed in chunks of up to @bufsize@. The stream ends
410-- as soon as EOF is encountered.
412{-# INLINE readWithBufferOf #-}
413readWithBufferOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
414readWithBufferOf chunkSize h = A.flattenArrays $ readChunksUpto chunkSize h
418-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
420-- > read = 'readByChunks' A.defaultChunkSize
421-- | Generate a stream of elements of the given type from a socket. The
422-- stream ends when EOF is encountered.
425{-# INLINE toBytes #-}
426toBytes :: (IsStream t, MonadIO m) => Socket -> t m Word8
427toBytes = AS.concat . toChunks
429-- | Unfolds the tuple @(bufsize, socket)@ into a byte stream, read requests
430-- to the socket are performed using buffers of @bufsize@.
433{-# INLINE readWithBufferOf #-}
434readWithBufferOf :: MonadIO m => Unfold m (Int, Socket) Word8
435readWithBufferOf = UF.concat readChunksWithBufferOf A.read
437-- | Unfolds a 'Socket' into a byte stream. IO requests to the socket are
438-- performed in sizes of
439-- 'Streamly.Internal.Memory.Array.Types.defaultChunkSize'.
443read :: MonadIO m => Unfold m Socket Word8
444read = UF.supplyFirst readWithBufferOf A.defaultChunkSize
446-------------------------------------------------------------------------------
448-------------------------------------------------------------------------------
450-- | Write a stream of arrays to a handle.
453{-# INLINE fromChunks #-}
454fromChunks :: (MonadIO m, Storable a)
455 => Socket -> SerialT m (Array a) -> m ()
456fromChunks h = S.mapM_ (liftIO . writeChunk h)
458-- | Write a stream of arrays to a socket. Each array in the stream is written
459-- to the socket as a separate IO request.
462{-# INLINE writeChunks #-}
463writeChunks :: (MonadIO m, Storable a) => Socket -> Fold m (Array a) ()
464writeChunks h = FL.drainBy (liftIO . writeChunk h)
466-- | @writeChunksWithBufferOf bufsize socket@ writes a stream of arrays
467-- to @socket@ after coalescing the adjacent arrays in chunks of @bufsize@.
468-- We never split an array, if a single array is bigger than the specified size
469-- it emitted as it is. Multiple arrays are coalesed as long as the total size
470-- remains below the specified size.
473{-# INLINE writeChunksWithBufferOf #-}
474writeChunksWithBufferOf :: (MonadIO m, Storable a)
475 => Int -> Socket -> Fold m (Array a) ()
476writeChunksWithBufferOf n h = lpackArraysChunksOf n (writeChunks h)
478-- | Write a stream of strings to a socket in Latin1 encoding. Output is
479-- flushed to the socket for each string.
483{-# INLINE writeStrings #-}
484writeStrings :: MonadIO m
485 => (SerialT m Char -> SerialT m Word8) -> Socket -> Fold m String ()
486writeStrings encode h =
487 FL.lmapM (IA.fromStream . encode . S.fromList) (writeChunks h)
489-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
492-- Note that if you use a chunk size less than 8K (GHC's default buffer
493-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
494-- do not want buffering to occur at GHC level as well. Same thing applies to
497-- | Like 'write' but provides control over the write buffer. Output will
498-- be written to the IO device as soon as we collect the specified number of
502{-# INLINE fromBytesWithBufferOf #-}
503fromBytesWithBufferOf :: MonadIO m => Int -> Socket -> SerialT m Word8 -> m ()
504fromBytesWithBufferOf n h m = fromChunks h $ AS.arraysOf n m
506-- | Write a byte stream to a socket. Accumulates the input in chunks of
507-- specified number of bytes before writing.
510{-# INLINE writeWithBufferOf #-}
511writeWithBufferOf :: MonadIO m => Int -> Socket -> Fold m Word8 ()
512writeWithBufferOf n h = FL.lchunksOf n (A.writeNUnsafe n) (writeChunks h)
514-- > write = 'writeWithBufferOf' A.defaultChunkSize
516-- | Write a byte stream to a file handle. Combines the bytes in chunks of size
517-- up to 'A.defaultChunkSize' before writing. Note that the write behavior
518-- depends on the 'IOMode' and the current seek position of the handle.
521{-# INLINE fromBytes #-}
522fromBytes :: MonadIO m => Socket -> SerialT m Word8 -> m ()
523fromBytes = fromBytesWithBufferOf A.defaultChunkSize
525-- | Write a byte stream to a socket. Accumulates the input in chunks of
526-- up to 'A.defaultChunkSize' bytes before writing.
529-- write = 'writeWithBufferOf' 'A.defaultChunkSize'
534write :: MonadIO m => Socket -> Fold m Word8 ()
535write = writeWithBufferOf A.defaultChunkSize
539write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
540write = toHandleWith A.defaultChunkSize
543-------------------------------------------------------------------------------
544-- IO with encoding/decoding Unicode characters
545-------------------------------------------------------------------------------
549-- > readUtf8 = decodeUtf8 . read
551-- Read a UTF8 encoded stream of unicode characters from a file handle.
554{-# INLINE readUtf8 #-}
555readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
556readUtf8 = decodeUtf8 . read
559-- > writeUtf8 h s = write h $ encodeUtf8 s
561-- Encode a stream of unicode characters to UTF8 and write it to the given file
562-- handle. Default block buffering applies to the writes.
565{-# INLINE writeUtf8 #-}
566writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
567writeUtf8 h s = write h $ encodeUtf8 s
569-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
570-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
571-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
572-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
576{-# INLINE writeUtf8ByLines #-}
577writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
578writeUtf8ByLines = undefined
580-- | Read UTF-8 lines from a file handle and apply the specified fold to each
581-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
584{-# INLINE readLines #-}
585readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
586readLines h f = foldLines (readUtf8 h) f
588-------------------------------------------------------------------------------
589-- Framing on a sequence
590-------------------------------------------------------------------------------
592-- | Read a stream from a file handle and split it into frames delimited by
593-- the specified sequence of elements. The supplied fold is applied on each
597{-# INLINE readFrames #-}
598readFrames :: (IsStream t, MonadIO m, Storable a)
599 => Array a -> Handle -> Fold m a b -> t m b
600readFrames = undefined -- foldFrames . read
602-- | Write a stream to the given file handle buffering up to frames separated
603-- by the given sequence or up to a maximum of @defaultChunkSize@.
606{-# INLINE writeByFrames #-}
607writeByFrames :: (IsStream t, MonadIO m, Storable a)
608 => Array a -> Handle -> t m a -> m ()
609writeByFrames = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list