2{-# LANGUAGE BangPatterns #-}
3{-# LANGUAGE MagicHash #-}
4{-# LANGUAGE RecordWildCards #-}
5{-# LANGUAGE UnboxedTuples #-}
10-- Module : Streamly.FileSystem.Handle
11-- Copyright : (c) 2018 Harendra Kumar
14-- Maintainer : harendra.kumar@gmail.com
15-- Stability : experimental
18-- Read and write streams and arrays to and from file handles. File handle IO
19-- APIs are quite similar to "Streamly.Mem.Array" read write APIs. In that
20-- regard, arrays can be considered as in-memory files or files can be
21-- considered as on-disk arrays.
23-- Control over file reading and writing behavior in terms of buffering,
24-- encoding, decoding is in the hands of the programmer, the 'TextEncoding',
25-- 'NewLineMode', and 'Buffering' options of the underlying handle provided by
26-- GHC are not needed and ignored.
28-- > import qualified Streamly.FileSystem.Handle as FH
31-- IO APIs are divided into two categories, sequential streaming IO APIs and
32-- random access IO APIs.
34module Streamly.FileSystem.Handle
36 -- * Sequential/Streaming IO
37 -- | Stream data to or from a file or device sequentially. When reading,
38 -- the stream is lazy and generated on-demand as the consumer consumes it.
39 -- Read IO requests to the IO device are performed in chunks limited to a
40 -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
41 -- documentation. One IO request may or may not read the full
42 -- chunk. If the whole stream is not consumed, it is possible that we may
43 -- read slightly more from the IO device than what the consumer needed.
44 -- Unless specified otherwise in the API, writes are collected into chunks
45 -- of @defaultChunkSize@ before they are written to the IO device.
47 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
48 -- including disks, files, memory devices, terminals, pipes, sockets and
49 -- fifos. While random access APIs work only for files or devices that have
50 -- random access or seek capability for example disks, memory devices.
51 -- Devices like terminals, pipes, sockets and fifos do not have random
54 -- ** Read From Handle
55 -- | 'TextEncoding', 'NewLineMode', and 'Buffering' options of the
56 -- underlying handle are ignored. The read occurs from the current seek
57 -- position of the file handle. The stream ends as soon as EOF is
75 -- | 'TextEncoding', 'NewLineMode', and 'Buffering' options of the
76 -- underlying handle are ignored. The write occurs from the current seek
77 -- position of the file handle. The write behavior depends on the 'IOMode'
89 , writeArraysPackedUpto
91 -- -- * Random Access (Seek)
92 -- -- | Unlike the streaming APIs listed above, these APIs apply to devices or
93 -- files that have random access or seek capability. This type of devices
94 -- include disks, files, memory devices and exclude terminals, pipes,
100 -- , readAt -- read from a given position to th end of file
101 -- , readSliceArrayUpto
102 -- , readSliceArrayOf
107 -- , writeAt -- start writing at the given position
112import Control.Monad.IO.Class (MonadIO(..))
113import Data.Word (Word8)
114import Foreign.ForeignPtr (withForeignPtr)
115import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
116import Foreign.Ptr (minusPtr, plusPtr)
117import Foreign.Storable (Storable(..))
118import GHC.ForeignPtr (mallocPlainForeignPtrBytes)
119import System.IO (Handle, hGetBufSome, hPutBuf)
120import Prelude hiding (read)
122import Streamly.Mem.Array.Types (Array(..))
123import Streamly.Streams.Serial (SerialT)
124import Streamly.Streams.StreamK.Type (IsStream, mkStream)
125import Streamly.Mem.Array.Types (defaultChunkSize, shrinkToFit)
126-- import Streamly.Fold (Fold)
127-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
129import qualified Streamly.Mem.Array as A
130import qualified Streamly.Prelude as S
131import qualified Streamly.Streams.StreamD.Type as D
133-------------------------------------------------------------------------------
135-------------------------------------------------------------------------------
137-- The following references may be useful to build an understanding about the
140-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
141-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
142-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
143-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
144-- https://www.w3.org/TR/FileAPI/ for http file API.
146-------------------------------------------------------------------------------
148-------------------------------------------------------------------------------
150-- | Read a 'ByteArray' from a file handle. If no data is available on the
151-- handle it blocks until some data becomes available. If data is available
152-- then it immediately returns that data without blocking. It reads a maximum
153-- of up to the size requested.
154{-# INLINABLE readArrayUpto #-}
155readArrayUpto :: Int -> Handle -> IO (Array Word8)
156readArrayUpto size h = do
157 ptr <- mallocPlainForeignPtrBytes size
158 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
159 withForeignPtr ptr $ \p -> do
160 n <- hGetBufSome h p size
163 , aEnd = p `plusPtr` n
164 , aBound = p `plusPtr` size
166 -- XXX shrink only if the diff is significant
169-------------------------------------------------------------------------------
171-------------------------------------------------------------------------------
173-- | Write an 'Array' to a file handle.
176{-# INLINABLE writeArray #-}
177writeArray :: Storable a => Handle -> Array a -> IO ()
178writeArray _ arr | A.length arr == 0 = return ()
179writeArray h Array{..} = withForeignPtr aStart $ \p -> hPutBuf h p aLen
182 let p = unsafeForeignPtrToPtr aStart
185-------------------------------------------------------------------------------
186-- Stream of Arrays IO
187-------------------------------------------------------------------------------
189-- | @readArraysOfUpto size h@ reads a stream of arrays from file handle @h@.
190-- The maximum size of a single array is specified by @size@. The actual size
191-- read may be less than or equal to @size@.
192{-# INLINABLE _readArraysOfUpto #-}
193_readArraysOfUpto :: (IsStream t, MonadIO m)
194 => Int -> Handle -> t m (Array Word8)
195_readArraysOfUpto size h = go
197 -- XXX use cons/nil instead
198 go = mkStream $ \_ yld _ stp -> do
199 arr <- liftIO $ readArrayUpto size h
204-- | @readArraysOfUpto size handle@ reads a stream of arrays from the file
205-- handle @handle@. The maximum size of a single array is limited to @size@.
208{-# INLINE_NORMAL readArraysOfUpto #-}
209readArraysOfUpto :: (IsStream t, MonadIO m) => Int -> Handle -> t m (Array Word8)
210readArraysOfUpto size h = D.fromStreamD (D.Stream step ())
212 {-# INLINE_LATE step #-}
214 arr <- liftIO $ readArrayUpto size h
220-- XXX read 'Array a' instead of Word8
222-- | @readArrays handle@ reads a stream of arrays from the specified file
223-- handle. The maximum size of a single array is limited to
224-- @defaultChunkSize@.
226-- > readArrays = readArraysOfUpto defaultChunkSize
229{-# INLINE readArrays #-}
230readArrays :: (IsStream t, MonadIO m) => Handle -> t m (Array Word8)
231readArrays = readArraysOfUpto defaultChunkSize
233-------------------------------------------------------------------------------
234-- Read File to Stream
235-------------------------------------------------------------------------------
237-- TODO for concurrent streams implement readahead IO. We can send multiple
238-- read requests at the same time. For serial case we can use async IO. We can
239-- also control the read throughput in mbps or IOPS.
241-- | @readByChunksUpto chunkSize handle@ reads a byte stream from a file
242-- handle, reads are performed in chunks of up to @chunkSize@.
245{-# INLINE readByChunksUpto #-}
246readByChunksUpto :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
247readByChunksUpto chunkSize h = A.flattenArrays $ readArraysOfUpto chunkSize h
250-- Generate a stream of elements of the given type from a file 'Handle'.
251-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
253-- > read = 'readByChunks' A.defaultChunkSize
254-- | Generate a byte stream from a file 'Handle'.
258read :: (IsStream t, MonadIO m) => Handle -> t m Word8
259read = A.flattenArrays . readArrays
261-------------------------------------------------------------------------------
263-------------------------------------------------------------------------------
265-- | Write a stream of arrays to a handle.
268{-# INLINE writeArrays #-}
269writeArrays :: (MonadIO m, Storable a) => Handle -> SerialT m (Array a) -> m ()
270writeArrays h m = S.mapM_ (liftIO . writeArray h) m
272-- | @writeArraysPackedUpto chunkSize handle stream@ writes a stream of arrays
273-- to @handle@ after coalescing the adjacent arrays in chunks of @chunkSize@.
274-- The chunk size is only a maximum and the actual writes could be smaller as
275-- we do not split the arrays to fit exactly to the specified size.
278{-# INLINE writeArraysPackedUpto #-}
279writeArraysPackedUpto :: (MonadIO m, Storable a)
280 => Int -> Handle -> SerialT m (Array a) -> m ()
281writeArraysPackedUpto n h xs = writeArrays h $ A.packArraysChunksOf n xs
283-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
286-- Note that if you use a chunk size less than 8K (GHC's default buffer
287-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
288-- do not want buffering to occur at GHC level as well. Same thing applies to
291-- | @writeByChunksOf chunkSize handle stream@ writes @stream@ to @handle@ in
292-- chunks of @chunkSize@. A write is performed to the IO device as soon as we
293-- collect the required input size.
296{-# INLINE writeByChunksOf #-}
297writeByChunksOf :: MonadIO m => Int -> Handle -> SerialT m Word8 -> m ()
298writeByChunksOf n h m = writeArrays h $ A.arraysOf n m
300-- > write = 'writeByChunksOf' A.defaultChunkSize
302-- | Write a byte stream to a file handle. Combines the bytes in chunks of
303-- up to 'A.defaultChunkSize' before writing.
307write :: MonadIO m => Handle -> SerialT m Word8 -> m ()
308write = writeByChunksOf defaultChunkSize
312write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
313write = toHandleWith A.defaultChunkSize
316-- XXX mmap a file into an array. This could be useful for in-place operations
317-- on a file. For example, we can quicksort the contents of a file by mmapping
320-------------------------------------------------------------------------------
321-- IO with encoding/decoding Unicode characters
322-------------------------------------------------------------------------------
326-- > readUtf8 = decodeUtf8 . read
328-- Read a UTF8 encoded stream of unicode characters from a file handle.
331{-# INLINE readUtf8 #-}
332readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
333readUtf8 = decodeUtf8 . read
336-- > writeUtf8 h s = write h $ encodeUtf8 s
338-- Encode a stream of unicode characters to UTF8 and write it to the given file
339-- handle. Default block buffering applies to the writes.
342{-# INLINE writeUtf8 #-}
343writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
344writeUtf8 h s = write h $ encodeUtf8 s
346-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
347-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
348-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
349-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
353{-# INLINE writeUtf8ByLines #-}
354writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
355writeUtf8ByLines = undefined
357-- | Read UTF-8 lines from a file handle and apply the specified fold to each
358-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
361{-# INLINE readLines #-}
362readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
363readLines h f = foldLines (readUtf8 h) f
365-------------------------------------------------------------------------------
366-- Framing on a sequence
367-------------------------------------------------------------------------------
369-- | Read a stream from a file handle and split it into frames delimited by
370-- the specified sequence of elements. The supplied fold is applied on each
374{-# INLINE readFrames #-}
375readFrames :: (IsStream t, MonadIO m, Storable a)
376 => Array a -> Handle -> Fold m a b -> t m b
377readFrames = undefined -- foldFrames . read
379-- | Write a stream to the given file handle buffering up to frames separated
380-- by the given sequence or up to a maximum of @defaultChunkSize@.
383{-# INLINE writeByFrames #-}
384writeByFrames :: (IsStream t, MonadIO m, Storable a)
385 => Array a -> Handle -> t m a -> m ()
386writeByFrames = undefined
388-------------------------------------------------------------------------------
390-------------------------------------------------------------------------------
392-- | Write collecting the input in sessions of n seconds or if chunkSize
394{-# INLINE writeByChunksOrSessionsOf #-}
395writeByChunksOrSessionsOf :: MonadIO m
396 => Int -> Double -> Handle -> SerialT m Word8 -> m ()
397writeByChunksOrSessionsOf chunkSize sessionSize h m = undefined
399-- | Write collecting the input in sessions of n seconds or if defaultChunkSize
401{-# INLINE writeBySessionsOf #-}
402writeBySessionsOf :: MonadIO m => Double -> Handle -> SerialT m Word8 -> m ()
403writeBySessionsOf n = writeByChunksOrSessionsOf defaultChunkSize n
405-------------------------------------------------------------------------------
406-- Random Access IO (Seek)
407-------------------------------------------------------------------------------
409-- XXX handles could be shared, so we may not want to use the handle state at
410-- all for these APIs. we can use pread and pwrite instead. On windows we will
411-- need to use readFile/writeFile with an offset argument.
413-------------------------------------------------------------------------------
415-- | Read the element at the given index treating the file as an array.
418{-# INLINE readIndex #-}
419readIndex :: Storable a => Handle -> Int -> Maybe a
420readIndex arr i = undefined
422-- NOTE: To represent a range to read we have chosen (start, size) instead of
423-- (start, end). This helps in removing the ambiguity of whether "end" is
424-- included in the range or not.
426-- We could avoid specifying the range to be read and instead use "take size"
427-- on the stream, but it may end up reading more and then consume it partially.
429-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
430-- from @handle@ starting at the offset @pos@ from the beginning of the file.
432-- Reads are performed in chunks of size @chunkSize@. For block devices, to
433-- avoid reading partial blocks @chunkSize@ must align with the block size of
434-- the underlying device. If the underlying block size is unknown, it is a good
435-- idea to keep it a multiple 4KiB. This API ensures that the start of each
436-- chunk is aligned with @chunkSize@ from second chunk onwards.
438{-# INLINE readSliceWith #-}
439readSliceWith :: (IsStream t, MonadIO m, Storable a)
440 => Int -> Handle -> Int -> Int -> t m a
441readSliceWith chunkSize h pos len = undefined
443-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
444-- at index @i@ and reading up to @count@ elements in the forward direction
445-- ending at the index @i + count - 1@.
448{-# INLINE readSlice #-}
449readSlice :: (IsStream t, MonadIO m, Storable a)
450 => Handle -> Int -> Int -> t m a
451readSlice = readSliceWith A.defaultChunkSize
453-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
454-- at index @i@ and reading up to @count@ elements in the reverse direction
455-- ending at the index @i - count + 1@.
458{-# INLINE readSliceRev #-}
459readSliceRev :: (IsStream t, MonadIO m, Storable a)
460 => Handle -> Int -> Int -> t m a
461readSliceRev h i count = undefined
463-- | Write the given element at the given index in the file.
466{-# INLINE writeIndex #-}
467writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
468writeIndex h i a = undefined
470-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
471-- starting at index @i@ and writing up to @count@ elements in the forward
472-- direction ending at the index @i + count - 1@.
475{-# INLINE writeSlice #-}
476writeSlice :: (IsStream t, Monad m, Storable a)
477 => Handle -> Int -> Int -> t m a -> m ()
478writeSlice h i len s = undefined
480-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
481-- starting at index @i@ and writing up to @count@ elements in the reverse
482-- direction ending at the index @i - count + 1@.
485{-# INLINE writeSliceRev #-}
486writeSliceRev :: (IsStream t, Monad m, Storable a)
487 => Handle -> Int -> Int -> t m a -> m ()
488writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list