2{-# LANGUAGE BangPatterns #-}
3{-# LANGUAGE MagicHash #-}
4{-# LANGUAGE RecordWildCards #-}
5{-# LANGUAGE UnboxedTuples #-}
10-- Module : Streamly.FileSystem.Handle
11-- Copyright : (c) 2018 Harendra Kumar
14-- Maintainer : harendra.kumar@gmail.com
15-- Stability : experimental
18-- Read and write streams and arrays to and from file handles. File handle IO
19-- APIs are quite similar to "Streamly.Mem.Array" read write APIs. In that
20-- regard, arrays can be considered as in-memory files or files can be
21-- considered as on-disk arrays. IO APIs are divided into two categories,
22-- sequential streaming IO APIs and random access IO APIs. Control over the
23-- file reading and writing behavior in terms of buffering, encoding, decoding
24-- is in the hands of the programmer, the 'TextEncoding', 'NewLineMode', and
25-- 'Buffering' options of the underlying handle provided by GHC are not needed
28-- > import qualified Streamly.FileSystem.Handle as FH
31module Streamly.FileSystem.Handle
34 -- | Stream data to or from a file or device sequentially. When reading,
35 -- the stream is lazy and generated on-demand as the consumer consumes it.
36 -- Read IO requests to the IO device are performed in chunks limited to a
37 -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
38 -- documentation. One IO request may or may not read the full
39 -- chunk. If the whole stream is not consumed, it is possible that we may
40 -- read slightly more from the IO device than what the consumer needed.
41 -- Unless specified otherwise in the API, writes are collected into chunks
42 -- of @defaultChunkSize@ before they are written to the IO device.
44 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
45 -- including disks, files, memory devices, terminals, pipes, sockets and
46 -- fifos. While random access APIs work only for files or devices that have
47 -- random access or seek capability for example disks, memory devices.
48 -- Devices like terminals, pipes, sockets and fifos do not have random
51 -- ** Read Handle to Stream
66 -- ** Write File from Stream
76 , writeArraysPackedUpto
78 -- -- * Random Access (Seek)
79 -- -- | Unlike the streaming APIs listed above, these APIs apply to devices or
80 -- files that have random access or seek capability. This type of devices
81 -- include disks, files, memory devices and exclude terminals, pipes,
87 -- , readAt -- read from a given position to th end of file
88 -- , readSliceArrayUpto
94 -- , writeAt -- start writing at the given position
99import Control.Monad.IO.Class (MonadIO(..))
100import Data.Word (Word8)
101import Foreign.ForeignPtr (withForeignPtr)
102import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
103import Foreign.Ptr (minusPtr, plusPtr)
104import Foreign.Storable (Storable(..))
105import GHC.ForeignPtr (mallocPlainForeignPtrBytes)
106import System.IO (Handle, hGetBufSome, hPutBuf)
107import Prelude hiding (read)
109import Streamly.Mem.Array.Types (Array(..))
110import Streamly.Streams.Serial (SerialT)
111import Streamly.Streams.StreamK.Type (IsStream, mkStream)
112import Streamly.Mem.Array.Types (defaultChunkSize, shrinkToFit)
113-- import Streamly.Fold (Fold)
114-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
116import qualified Streamly.Mem.Array as A
117import qualified Streamly.Prelude as S
118import qualified Streamly.Streams.StreamD.Type as D
120-------------------------------------------------------------------------------
122-------------------------------------------------------------------------------
124-- The following references may be useful to build an understanding about the
127-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
128-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
129-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
130-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
131-- https://www.w3.org/TR/FileAPI/ for http file API.
133-------------------------------------------------------------------------------
135-------------------------------------------------------------------------------
137-- | Read a 'ByteArray' from a file handle. If no data is available on the
138-- handle it blocks until some data becomes available. If data is available
139-- then it immediately returns that data without blocking. It reads a maximum
140-- of up to the size requested.
141{-# INLINABLE readArrayUpto #-}
142readArrayUpto :: Int -> Handle -> IO (Array Word8)
143readArrayUpto size h = do
144 ptr <- mallocPlainForeignPtrBytes size
145 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
146 withForeignPtr ptr $ \p -> do
147 n <- hGetBufSome h p size
150 , aEnd = p `plusPtr` n
151 , aBound = p `plusPtr` size
153 -- XXX shrink only if the diff is significant
156-------------------------------------------------------------------------------
158-------------------------------------------------------------------------------
160-- | Write an Array to a file handle.
163{-# INLINABLE writeArray #-}
164writeArray :: Storable a => Handle -> Array a -> IO ()
165writeArray _ arr | A.length arr == 0 = return ()
166writeArray h Array{..} = withForeignPtr aStart $ \p -> hPutBuf h p aLen
169 let p = unsafeForeignPtrToPtr aStart
172-------------------------------------------------------------------------------
173-- Stream of Arrays IO
174-------------------------------------------------------------------------------
176-- | @readArraysOfUpto size h@ reads a stream of arrays from file handle @h@.
177-- The maximum size of a single array is specified by @size@. The actual size
178-- read may be less than or equal to @size@.
179{-# INLINABLE _readArraysOfUpto #-}
180_readArraysOfUpto :: (IsStream t, MonadIO m)
181 => Int -> Handle -> t m (Array Word8)
182_readArraysOfUpto size h = go
184 -- XXX use cons/nil instead
185 go = mkStream $ \_ yld _ stp -> do
186 arr <- liftIO $ readArrayUpto size h
191{-# INLINE_NORMAL readArraysOfUpto #-}
192readArraysOfUpto :: (IsStream t, MonadIO m) => Int -> Handle -> t m (Array Word8)
193readArraysOfUpto size h = D.fromStreamD (D.Stream step ())
195 {-# INLINE_LATE step #-}
197 arr <- liftIO $ readArrayUpto size h
203-- XXX read 'Array a' instead of Word8
205-- | @readArrays h@ reads a stream of arrays from file handle @h@.
206-- The maximum size of a single array is limited to @defaultChunkSize@.
207-- 'readArrays' ignores the prevailing 'TextEncoding' and 'NewlineMode'
210-- > readArrays = readArraysOfUpto defaultChunkSize
213{-# INLINE readArrays #-}
214readArrays :: (IsStream t, MonadIO m) => Handle -> t m (Array Word8)
215readArrays = readArraysOfUpto defaultChunkSize
217-------------------------------------------------------------------------------
218-- Read File to Stream
219-------------------------------------------------------------------------------
221-- TODO for concurrent streams implement readahead IO. We can send multiple
222-- read requests at the same time. For serial case we can use async IO. We can
223-- also control the read throughput in mbps or IOPS.
225-- | @readByChunksUpto chunkSize handle@ reads a byte stream from a file
226-- handle, reads are performed in chunks of up to @chunkSize@. The stream ends
227-- as soon as EOF is encountered.
229{-# INLINE readByChunksUpto #-}
230readByChunksUpto :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
231readByChunksUpto chunkSize h = A.flattenArrays $ readArraysOfUpto chunkSize h
234-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
236-- > read = 'readByChunks' A.defaultChunkSize
237-- | Generate a stream of elements of the given type from a file 'Handle'. The
238-- stream ends when EOF is encountered.
242read :: (IsStream t, MonadIO m) => Handle -> t m Word8
243read = A.flattenArrays . readArrays
245-------------------------------------------------------------------------------
247-------------------------------------------------------------------------------
249-- | Write a stream of arrays to a handle.
252{-# INLINE writeArrays #-}
253writeArrays :: (MonadIO m, Storable a) => Handle -> SerialT m (Array a) -> m ()
254writeArrays h m = S.mapM_ (liftIO . writeArray h) m
256-- | Write a stream of arrays to a handle after coalescing them in chunks of
257-- specified size. The chunk size is only a maximum and the actual writes could
258-- be smaller than that as we do not split the arrays to fit them to the
262{-# INLINE writeArraysPackedUpto #-}
263writeArraysPackedUpto :: (MonadIO m, Storable a)
264 => Int -> Handle -> SerialT m (Array a) -> m ()
265writeArraysPackedUpto n h xs = writeArrays h $ A.packArraysChunksOf n xs
267-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
270-- Note that if you use a chunk size less than 8K (GHC's default buffer
271-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
272-- do not want buffering to occur at GHC level as well. Same thing applies to
275-- | Like 'write' but provides control over the write buffer. Output will
276-- be written to the IO device as soon as we collect the specified number of
280{-# INLINE writeByChunksOf #-}
281writeByChunksOf :: MonadIO m => Int -> Handle -> SerialT m Word8 -> m ()
282writeByChunksOf n h m = writeArrays h $ A.arraysOf n m
284-- > write = 'writeByChunksOf' A.defaultChunkSize
286-- | Write a byte stream to a file handle. Combines the bytes in chunks of size
287-- up to 'A.defaultChunkSize' before writing. Note that the write behavior
288-- depends on the 'IOMode' and the current seek position of the handle.
292write :: MonadIO m => Handle -> SerialT m Word8 -> m ()
293write = writeByChunksOf defaultChunkSize
297write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
298write = toHandleWith A.defaultChunkSize
301-- XXX mmap a file into an array. This could be useful for in-place operations
302-- on a file. For example, we can quicksort the contents of a file by mmapping
305-------------------------------------------------------------------------------
306-- IO with encoding/decoding Unicode characters
307-------------------------------------------------------------------------------
311-- > readUtf8 = decodeUtf8 . read
313-- Read a UTF8 encoded stream of unicode characters from a file handle.
316{-# INLINE readUtf8 #-}
317readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
318readUtf8 = decodeUtf8 . read
321-- > writeUtf8 h s = write h $ encodeUtf8 s
323-- Encode a stream of unicode characters to UTF8 and write it to the given file
324-- handle. Default block buffering applies to the writes.
327{-# INLINE writeUtf8 #-}
328writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
329writeUtf8 h s = write h $ encodeUtf8 s
331-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
332-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
333-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
334-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
338{-# INLINE writeUtf8ByLines #-}
339writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
340writeUtf8ByLines = undefined
342-- | Read UTF-8 lines from a file handle and apply the specified fold to each
343-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
346{-# INLINE readLines #-}
347readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
348readLines h f = foldLines (readUtf8 h) f
350-------------------------------------------------------------------------------
351-- Framing on a sequence
352-------------------------------------------------------------------------------
354-- | Read a stream from a file handle and split it into frames delimited by
355-- the specified sequence of elements. The supplied fold is applied on each
359{-# INLINE readFrames #-}
360readFrames :: (IsStream t, MonadIO m, Storable a)
361 => Array a -> Handle -> Fold m a b -> t m b
362readFrames = undefined -- foldFrames . read
364-- | Write a stream to the given file handle buffering up to frames separated
365-- by the given sequence or up to a maximum of @defaultChunkSize@.
368{-# INLINE writeByFrames #-}
369writeByFrames :: (IsStream t, MonadIO m, Storable a)
370 => Array a -> Handle -> t m a -> m ()
371writeByFrames = undefined
373-------------------------------------------------------------------------------
374-- Random Access IO (Seek)
375-------------------------------------------------------------------------------
377-- XXX handles could be shared, so we may not want to use the handle state at
378-- all for these APIs. we can use pread and pwrite instead. On windows we will
379-- need to use readFile/writeFile with an offset argument.
381-------------------------------------------------------------------------------
383-- | Read the element at the given index treating the file as an array.
386{-# INLINE readIndex #-}
387readIndex :: Storable a => Handle -> Int -> Maybe a
388readIndex arr i = undefined
390-- NOTE: To represent a range to read we have chosen (start, size) instead of
391-- (start, end). This helps in removing the ambiguity of whether "end" is
392-- included in the range or not.
394-- We could avoid specifying the range to be read and instead use "take size"
395-- on the stream, but it may end up reading more and then consume it partially.
397-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
398-- from @handle@ starting at the offset @pos@ from the beginning of the file.
400-- Reads are performed in chunks of size @chunkSize@. For block devices, to
401-- avoid reading partial blocks @chunkSize@ must align with the block size of
402-- the underlying device. If the underlying block size is unknown, it is a good
403-- idea to keep it a multiple 4KiB. This API ensures that the start of each
404-- chunk is aligned with @chunkSize@ from second chunk onwards.
406{-# INLINE readSliceWith #-}
407readSliceWith :: (IsStream t, MonadIO m, Storable a)
408 => Int -> Handle -> Int -> Int -> t m a
409readSliceWith chunkSize h pos len = undefined
411-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
412-- at index @i@ and reading up to @count@ elements in the forward direction
413-- ending at the index @i + count - 1@.
416{-# INLINE readSlice #-}
417readSlice :: (IsStream t, MonadIO m, Storable a)
418 => Handle -> Int -> Int -> t m a
419readSlice = readSliceWith A.defaultChunkSize
421-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
422-- at index @i@ and reading up to @count@ elements in the reverse direction
423-- ending at the index @i - count + 1@.
426{-# INLINE readSliceRev #-}
427readSliceRev :: (IsStream t, MonadIO m, Storable a)
428 => Handle -> Int -> Int -> t m a
429readSliceRev h i count = undefined
431-- | Write the given element at the given index in the file.
434{-# INLINE writeIndex #-}
435writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
436writeIndex h i a = undefined
438-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
439-- starting at index @i@ and writing up to @count@ elements in the forward
440-- direction ending at the index @i + count - 1@.
443{-# INLINE writeSlice #-}
444writeSlice :: (IsStream t, Monad m, Storable a)
445 => Handle -> Int -> Int -> t m a -> m ()
446writeSlice h i len s = undefined
448-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
449-- starting at index @i@ and writing up to @count@ elements in the reverse
450-- direction ending at the index @i - count + 1@.
453{-# INLINE writeSliceRev #-}
454writeSliceRev :: (IsStream t, Monad m, Storable a)
455 => Handle -> Int -> Int -> t m a -> m ()
456writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list