2{-# LANGUAGE BangPatterns #-}
3{-# LANGUAGE MagicHash #-}
4{-# LANGUAGE RecordWildCards #-}
5{-# LANGUAGE UnboxedTuples #-}
10-- Module : Streamly.FileSystem.File
11-- Copyright : (c) 2018 Harendra Kumar
14-- Maintainer : harendra.kumar@gmail.com
15-- Stability : experimental
18-- Read and write streams and arrays to and from files. File IO APIs are quite
19-- similar to "Streamly.Mem.Array" read write APIs. In that regard, arrays can
20-- be considered as in-memory files or files can be considered as on-disk
21-- arrays. IO APIs are divided into two categories, sequential streaming IO
22-- APIs and random access IO APIs. Control over the file reading and writing
23-- behavior in terms of buffering, encoding, decoding is in the hands of the
24-- programmer, the 'TextEncoding', 'NewLineMode', and 'Buffering' options of
25-- the underlying handle provided by GHC are not needed and ignored.
27-- > import qualified Streamly.FileSystem.File as File
30module Streamly.FileSystem.File
33 -- | Stream data to or from a file or device sequentially. When reading,
34 -- the stream is lazy and generated on-demand as the consumer consumes it.
35 -- Read IO requests to the IO device are performed in chunks limited to a
36 -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
37 -- documentation. One IO request may or may not read the full
38 -- chunk. If the whole stream is not consumed, it is possible that we may
39 -- read slightly more from the IO device than what the consumer needed.
40 -- Unless specified otherwise in the API, writes are collected into chunks
41 -- of @defaultChunkSize@ before they are written to the IO device.
43 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
44 -- including disks, files, memory devices, terminals, pipes, sockets and
45 -- fifos. While random access APIs work only for files or devices that have
46 -- random access or seek capability for example disks, memory devices.
47 -- Devices like terminals, pipes, sockets and fifos do not have random
50 -- TODO file path based APIs
54 -- ** Read File to Stream
69 -- ** Write File from Stream
80 -- -- * Random Access (Seek)
81 -- -- | Unlike the streaming APIs listed above, these APIs apply to devices or
82 -- files that have random access or seek capability. This type of devices
83 -- include disks, files, memory devices and exclude terminals, pipes,
89 -- , readAt -- read from a given position to th end of file
90 -- , readSliceArrayUpto
96 -- , writeAt -- start writing at the given position
101import Control.Monad.IO.Class (MonadIO(..))
102import Data.Word (Word8)
103import Foreign.ForeignPtr (withForeignPtr)
104import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
105import Foreign.Ptr (minusPtr, plusPtr)
106import Foreign.Storable (Storable(..))
107import GHC.ForeignPtr (mallocPlainForeignPtrBytes)
108import System.IO (Handle, hGetBufSome, hPutBuf)
109import Prelude hiding (read)
111import Streamly.Mem.Array.Types (Array(..))
112import Streamly.Streams.Serial (SerialT)
113import Streamly.Streams.StreamK.Type (IsStream, mkStream)
114-- import Streamly.Fold (Fold)
115-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
117import qualified Streamly.Mem.Array as A
118import qualified Streamly.Mem.Array.Types as A hiding (flattenArrays)
119import qualified Streamly.Prelude as S
121-------------------------------------------------------------------------------
123-------------------------------------------------------------------------------
125-- The following references may be useful to build an understanding about the
128-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
129-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
130-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
131-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
132-- https://www.w3.org/TR/FileAPI/ for http file API.
134-------------------------------------------------------------------------------
136-------------------------------------------------------------------------------
138-- | Read a 'ByteArray' from a file handle. If no data is available on the
139-- handle it blocks until some data becomes available. If data is available
140-- then it immediately returns that data without blocking. It reads a maximum
141-- of up to the size requested.
142{-# INLINABLE readArrayUpto #-}
143readArrayUpto :: Int -> Handle -> IO (Array Word8)
144readArrayUpto size h = do
145 ptr <- mallocPlainForeignPtrBytes size
146 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
147 withForeignPtr ptr $ \p -> do
148 n <- hGetBufSome h p size
151 , aEnd = p `plusPtr` n
152 , aBound = p `plusPtr` size
154 -- XXX shrink only if the diff is significant
157-------------------------------------------------------------------------------
159-------------------------------------------------------------------------------
161-- | Write an Array to a file handle.
164{-# INLINABLE writeArray #-}
165writeArray :: Storable a => Handle -> Array a -> IO ()
166writeArray _ arr | A.length arr == 0 = return ()
167writeArray h Array{..} = withForeignPtr aStart $ \p -> hPutBuf h p aLen
170 let p = unsafeForeignPtrToPtr aStart
173-------------------------------------------------------------------------------
174-- Stream of Arrays IO
175-------------------------------------------------------------------------------
177-- | @readArraysUpto size h@ reads a stream of arrays from file handle @h@.
178-- The maximum size of a single array is specified by @size@. The actual size
179-- read may be less than or equal to @size@.
180{-# INLINABLE readArraysUpto #-}
181readArraysUpto :: (IsStream t, MonadIO m)
182 => Int -> Handle -> t m (Array Word8)
183readArraysUpto size h = go
185 -- XXX use cons/nil instead
186 go = mkStream $ \_ yld _ stp -> do
187 arr <- liftIO $ readArrayUpto size h
192-- XXX read 'Array a' instead of Word8
194-- | @readArrays h@ reads a stream of arrays from file handle @h@.
195-- The maximum size of a single array is limited to @defaultChunkSize@.
196-- 'readArrays' ignores the prevailing 'TextEncoding' and 'NewlineMode'
199-- > readArrays = readArraysUpto defaultChunkSize
202{-# INLINE readArrays #-}
203readArrays :: (IsStream t, MonadIO m) => Handle -> t m (Array Word8)
204readArrays = readArraysUpto A.defaultChunkSize
206-------------------------------------------------------------------------------
207-- Read File to Stream
208-------------------------------------------------------------------------------
210-- TODO for concurrent streams implement readahead IO. We can send multiple
211-- read requests at the same time. For serial case we can use async IO. We can
212-- also control the read throughput in mbps or IOPS.
215-- | @readByChunksUpto chunkSize handle@ reads a byte stream from a file
216-- handle, reads are performed in chunks of up to @chunkSize@. The stream ends
217-- as soon as EOF is encountered.
219{-# INLINE readByChunksUpto #-}
220readByChunksUpto :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
221readByChunksUpto chunkSize h = A.flattenArrays $ readArraysUpto chunkSize h
225-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
227-- > read = 'readByChunks' A.defaultChunkSize
228-- | Generate a stream of elements of the given type from a file 'Handle'. The
229-- stream ends when EOF is encountered.
233read :: (IsStream t, MonadIO m) => Handle -> t m Word8
234read = A.flattenArrays . readArrays
236-------------------------------------------------------------------------------
238-------------------------------------------------------------------------------
240-- | Write a stream of arrays to a handle.
243{-# INLINE writeArrays #-}
244writeArrays :: (MonadIO m, Storable a) => Handle -> SerialT m (Array a) -> m ()
245writeArrays h m = S.mapM_ (liftIO . writeArray h) m
247-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
250-- Note that if you use a chunk size less than 8K (GHC's default buffer
251-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
252-- do not want buffering to occur at GHC level as well. Same thing applies to
255-- | Like 'write' but provides control over the write buffer. Output will
256-- be written to the IO device as soon as we collect the specified number of
260{-# INLINE writeByChunks #-}
261writeByChunks :: MonadIO m => Int -> Handle -> SerialT m Word8 -> m ()
262writeByChunks n h m = writeArrays h $ A.arraysOf n m
264-- > write = 'writeByChunks' A.defaultChunkSize
266-- | Write a byte stream to a file handle. Combines the bytes in chunks of size
267-- up to 'A.defaultChunkSize' before writing. Note that the write behavior
268-- depends on the 'IOMode' and the current seek position of the handle.
272write :: MonadIO m => Handle -> SerialT m Word8 -> m ()
273write = writeByChunks A.defaultChunkSize
277write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
278write = toHandleWith A.defaultChunkSize
281-- XXX mmap a file into an array. This could be useful for in-place operations
282-- on a file. For example, we can quicksort the contents of a file by mmapping
285-------------------------------------------------------------------------------
286-- IO with encoding/decoding Unicode characters
287-------------------------------------------------------------------------------
291-- > readUtf8 = decodeUtf8 . read
293-- Read a UTF8 encoded stream of unicode characters from a file handle.
296{-# INLINE readUtf8 #-}
297readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
298readUtf8 = decodeUtf8 . read
301-- > writeUtf8 h s = write h $ encodeUtf8 s
303-- Encode a stream of unicode characters to UTF8 and write it to the given file
304-- handle. Default block buffering applies to the writes.
307{-# INLINE writeUtf8 #-}
308writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
309writeUtf8 h s = write h $ encodeUtf8 s
311-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
312-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
313-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
314-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
318{-# INLINE writeUtf8ByLines #-}
319writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
320writeUtf8ByLines = undefined
322-- | Read UTF-8 lines from a file handle and apply the specified fold to each
323-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
326{-# INLINE readLines #-}
327readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
328readLines h f = foldLines (readUtf8 h) f
330-------------------------------------------------------------------------------
331-- Framing on a sequence
332-------------------------------------------------------------------------------
334-- | Read a stream from a file handle and split it into frames delimited by
335-- the specified sequence of elements. The supplied fold is applied on each
339{-# INLINE readFrames #-}
340readFrames :: (IsStream t, MonadIO m, Storable a)
341 => Array a -> Handle -> Fold m a b -> t m b
342readFrames = undefined -- foldFrames . read
344-- | Write a stream to the given file handle buffering up to frames separated
345-- by the given sequence or up to a maximum of @defaultChunkSize@.
348{-# INLINE writeByFrames #-}
349writeByFrames :: (IsStream t, MonadIO m, Storable a)
350 => Array a -> Handle -> t m a -> m ()
351writeByFrames = undefined
353-------------------------------------------------------------------------------
354-- Random Access IO (Seek)
355-------------------------------------------------------------------------------
357-- XXX handles could be shared, so we may not want to use the handle state at
358-- all for these APIs. we can use pread and pwrite instead. On windows we will
359-- need to use readFile/writeFile with an offset argument.
361-------------------------------------------------------------------------------
363-- | Read the element at the given index treating the file as an array.
366{-# INLINE readIndex #-}
367readIndex :: Storable a => Handle -> Int -> Maybe a
368readIndex arr i = undefined
370-- NOTE: To represent a range to read we have chosen (start, size) instead of
371-- (start, end). This helps in removing the ambiguity of whether "end" is
372-- included in the range or not.
374-- We could avoid specifying the range to be read and instead use "take size"
375-- on the stream, but it may end up reading more and then consume it partially.
377-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
378-- from @handle@ starting at the offset @pos@ from the beginning of the file.
380-- Reads are performed in chunks of size @chunkSize@. For block devices, to
381-- avoid reading partial blocks @chunkSize@ must align with the block size of
382-- the underlying device. If the underlying block size is unknown, it is a good
383-- idea to keep it a multiple 4KiB. This API ensures that the start of each
384-- chunk is aligned with @chunkSize@ from second chunk onwards.
386{-# INLINE readSliceWith #-}
387readSliceWith :: (IsStream t, MonadIO m, Storable a)
388 => Int -> Handle -> Int -> Int -> t m a
389readSliceWith chunkSize h pos len = undefined
391-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
392-- at index @i@ and reading up to @count@ elements in the forward direction
393-- ending at the index @i + count - 1@.
396{-# INLINE readSlice #-}
397readSlice :: (IsStream t, MonadIO m, Storable a)
398 => Handle -> Int -> Int -> t m a
399readSlice = readSliceWith A.defaultChunkSize
401-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
402-- at index @i@ and reading up to @count@ elements in the reverse direction
403-- ending at the index @i - count + 1@.
406{-# INLINE readSliceRev #-}
407readSliceRev :: (IsStream t, MonadIO m, Storable a)
408 => Handle -> Int -> Int -> t m a
409readSliceRev h i count = undefined
411-- | Write the given element at the given index in the file.
414{-# INLINE writeIndex #-}
415writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
416writeIndex h i a = undefined
418-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
419-- starting at index @i@ and writing up to @count@ elements in the forward
420-- direction ending at the index @i + count - 1@.
423{-# INLINE writeSlice #-}
424writeSlice :: (IsStream t, Monad m, Storable a)
425 => Handle -> Int -> Int -> t m a -> m ()
426writeSlice h i len s = undefined
428-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
429-- starting at index @i@ and writing up to @count@ elements in the reverse
430-- direction ending at the index @i - count + 1@.
433{-# INLINE writeSliceRev #-}
434writeSliceRev :: (IsStream t, Monad m, Storable a)
435 => Handle -> Int -> Int -> t m a -> m ()
436writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list