4-- Module : Streamly.Internal.FileSystem.File
5-- Copyright : (c) 2019 Composewell Technologies
8-- Maintainer : streamly@composewell.com
9-- Stability : pre-release
12-- Read and write streams and arrays to and from files specified by their paths
13-- in the file system. Unlike the handle based APIs which can have a read/write
14-- session consisting of multiple reads and writes to the handle, these APIs
15-- are one shot read or write APIs. These APIs open the file handle, perform
16-- the requested operation and close the handle. Thease are safer compared to
17-- the handle based APIs as there is no possibility of a file descriptor
20-- > import qualified Streamly.Internal.FileSystem.File as File
23module Streamly.Internal.FileSystem.File
26 -- | Stream data to or from a file or device sequentially. When reading,
27 -- the stream is lazy and generated on-demand as the consumer consumes it.
28 -- Read IO requests to the IO device are performed in chunks limited to a
29 -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
30 -- documentation. One IO request may or may not read the full
31 -- chunk. If the whole stream is not consumed, it is possible that we may
32 -- read slightly more from the IO device than what the consumer needed.
33 -- Unless specified otherwise in the API, writes are collected into chunks
34 -- of @defaultChunkSize@ before they are written to the IO device.
36 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
37 -- including disks, files, memory devices, terminals, pipes, sockets and
38 -- fifos. While random access APIs work only for files or devices that have
39 -- random access or seek capability for example disks, memory devices.
40 -- Devices like terminals, pipes, sockets and fifos do not have random
43 -- ** File IO Using Handle
57 , readChunksWithBufferOf
58 , readChunksFromToWith
61 , toChunksWithBufferOf
72 , fromBytesWithBufferOf
88import Control.Monad.Catch (MonadCatch)
89import Control.Monad.IO.Class (MonadIO(..))
90import Data.Word (Word8)
91import Foreign.Storable (Storable(..))
92import System.IO (Handle, openFile, IOMode(..), hClose)
93import Prelude hiding (read)
95import qualified Control.Monad.Catch as MC
96import qualified System.IO as SIO
98import Streamly.Internal.Control.Concurrent (MonadAsync)
99import Streamly.Internal.Data.Fold.Type (Fold(..))
100import Streamly.Internal.Data.Unfold.Type (Unfold(..))
101import Streamly.Internal.Data.Array.Foreign.Type (Array(..), writeNUnsafe)
102import Streamly.Internal.Data.Stream.Serial (SerialT)
103import Streamly.Internal.Data.Stream.IsStream.Type (IsStream)
104-- import Streamly.Data.Fold (Fold)
105-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
106import Streamly.Internal.System.IO (defaultChunkSize)
108import qualified Streamly.Internal.Data.Array.Foreign as A
109import qualified Streamly.Internal.Data.Fold.Type as FL
110import qualified Streamly.Internal.Data.Unfold as UF
111import qualified Streamly.Internal.FileSystem.Handle as FH
112import qualified Streamly.Internal.Data.Array.Stream.Foreign as AS
113import qualified Streamly.Internal.Data.Stream.IsStream as S
115-------------------------------------------------------------------------------
117-------------------------------------------------------------------------------
119-- The following references may be useful to build an understanding about the
122-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
123-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
124-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
125-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
126-- https://www.w3.org/TR/FileAPI/ for http file API.
128-------------------------------------------------------------------------------
130-------------------------------------------------------------------------------
132-- | @'withFile' name mode act@ opens a file using 'openFile' and passes
133-- the resulting handle to the computation @act@. The handle will be
134-- closed on exit from 'withFile', whether by normal termination or by
135-- raising an exception. If closing the handle raises an exception, then
136-- this exception will be raised by 'withFile' rather than any exception
141{-# INLINE withFile #-}
142withFile :: (IsStream t, MonadCatch m, MonadAsync m)
143 => FilePath -> IOMode -> (Handle -> t m a) -> t m a
144withFile file mode = S.bracket (liftIO $ openFile file mode) (liftIO . hClose)
146-- | Transform an 'Unfold' from a 'Handle' to an unfold from a 'FilePath'. The
147-- resulting unfold opens a handle in 'ReadMode', uses it using the supplied
148-- unfold and then makes sure that the handle is closed on normal termination
149-- or in case of an exception. If closing the handle raises an exception, then
150-- this exception will be raised by 'usingFile'.
154{-# INLINABLE usingFile #-}
155usingFile :: (MonadCatch m, MonadAsync m)
156 => Unfold m Handle a -> Unfold m FilePath a
158 UF.bracket (\file -> liftIO $ openFile file ReadMode)
161{-# INLINABLE usingFile2 #-}
162usingFile2 :: (MonadCatch m, MonadAsync m)
163 => Unfold m (x, Handle) a -> Unfold m (x, FilePath) a
164usingFile2 = UF.bracket before after
168 before (x, file) = do
169 h <- liftIO $ openFile file ReadMode
172 after (_, h) = liftIO $ hClose h
174{-# INLINABLE usingFile3 #-}
175usingFile3 :: (MonadCatch m, MonadAsync m)
176 => Unfold m (x, y, z, Handle) a -> Unfold m (x, y, z, FilePath) a
177usingFile3 = UF.bracket before after
181 before (x, y, z, file) = do
182 h <- liftIO $ openFile file ReadMode
185 after (_, _, _, h) = liftIO $ hClose h
187-------------------------------------------------------------------------------
189-------------------------------------------------------------------------------
193-------------------------------------------------------------------------------
195-------------------------------------------------------------------------------
197-- | Write an array to a file. Overwrites the file if it exists.
200{-# INLINABLE putChunk #-}
201putChunk :: Storable a => FilePath -> Array a -> IO ()
202putChunk file arr = SIO.withFile file WriteMode (`FH.putChunk` arr)
204-- | append an array to a file.
207{-# INLINABLE appendArray #-}
208appendArray :: Storable a => FilePath -> Array a -> IO ()
209appendArray file arr = SIO.withFile file AppendMode (`FH.putChunk` arr)
211-------------------------------------------------------------------------------
212-- Stream of Arrays IO
213-------------------------------------------------------------------------------
215-- | @toChunksWithBufferOf size file@ reads a stream of arrays from file @file@.
216-- The maximum size of a single array is specified by @size@. The actual size
217-- read may be less than or equal to @size@.
218{-# INLINABLE toChunksWithBufferOf #-}
219toChunksWithBufferOf :: (IsStream t, MonadCatch m, MonadAsync m)
220 => Int -> FilePath -> t m (Array Word8)
221toChunksWithBufferOf size file =
222 withFile file ReadMode (FH.toChunksWithBufferOf size)
224-- XXX read 'Array a' instead of Word8
226-- | @toChunks file@ reads a stream of arrays from file @file@.
227-- The maximum size of a single array is limited to @defaultChunkSize@. The
228-- actual size read may be less than @defaultChunkSize@.
230-- > toChunks = toChunksWithBufferOf defaultChunkSize
233{-# INLINE toChunks #-}
234toChunks :: (IsStream t, MonadCatch m, MonadAsync m)
235 => FilePath -> t m (Array Word8)
236toChunks = toChunksWithBufferOf defaultChunkSize
238-------------------------------------------------------------------------------
239-- Read File to Stream
240-------------------------------------------------------------------------------
242-- TODO for concurrent streams implement readahead IO. We can send multiple
243-- read requests at the same time. For serial case we can use async IO. We can
244-- also control the read throughput in mbps or IOPS.
246-- | Unfold the tuple @(bufsize, filepath)@ into a stream of 'Word8' arrays.
247-- Read requests to the IO device are performed using a buffer of size
248-- @bufsize@. The size of an array in the resulting stream is always less than
249-- or equal to @bufsize@.
253{-# INLINE readChunksWithBufferOf #-}
254readChunksWithBufferOf :: (MonadCatch m, MonadAsync m)
255 => Unfold m (Int, FilePath) (Array Word8)
256readChunksWithBufferOf = usingFile2 FH.readChunksWithBufferOf
258-- | Unfold the tuple @(from, to, bufsize, filepath)@ into a stream
260-- Read requests to the IO device are performed using a buffer of size
261-- @bufsize@ starting from absolute offset of @from@ till the absolute
262-- position of @to@. The size of an array in the resulting stream is always
263-- less than or equal to @bufsize@.
266{-# INLINE readChunksFromToWith #-}
267readChunksFromToWith :: (MonadCatch m, MonadAsync m) =>
268 Unfold m (Int, Int, Int, FilePath) (Array Word8)
269readChunksFromToWith = usingFile3 FH.readChunksFromToWith
271-- | Unfolds a 'FilePath' into a stream of 'Word8' arrays. Requests to the IO
272-- device are performed using a buffer of size
273-- 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize'. The
274-- size of arrays in the resulting stream are therefore less than or equal to
275-- 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize'.
278{-# INLINE readChunks #-}
279readChunks :: (MonadCatch m, MonadAsync m) => Unfold m FilePath (Array Word8)
280readChunks = usingFile FH.readChunks
282-- | Unfolds the tuple @(bufsize, filepath)@ into a byte stream, read requests
283-- to the IO device are performed using buffers of @bufsize@.
286{-# INLINE readWithBufferOf #-}
287readWithBufferOf :: (MonadCatch m, MonadAsync m) => Unfold m (Int, FilePath) Word8
288readWithBufferOf = usingFile2 FH.readWithBufferOf
290-- | Unfolds a file path into a byte stream. IO requests to the device are
291-- performed in sizes of
292-- 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize'.
296read :: (MonadCatch m, MonadAsync m) => Unfold m FilePath Word8
297read = UF.many (usingFile FH.readChunks) A.read
299-- | Generate a stream of bytes from a file specified by path. The stream ends
300-- when EOF is encountered. File is locked using multiple reader and single
301-- writer locking mode.
305{-# INLINE toBytes #-}
306toBytes :: (IsStream t, MonadCatch m, MonadAsync m) => FilePath -> t m Word8
307toBytes file = AS.concat $ withFile file ReadMode FH.toChunks
310-- | Generate a stream of elements of the given type from a file 'Handle'. The
311-- stream ends when EOF is encountered. File is not locked for exclusive reads,
312-- writers can keep writing to the file.
315{-# INLINE readShared #-}
316readShared :: (IsStream t, MonadIO m) => Handle -> t m Word8
317readShared = undefined
320-------------------------------------------------------------------------------
322-------------------------------------------------------------------------------
324{-# INLINE fromChunksMode #-}
325fromChunksMode :: (MonadAsync m, MonadCatch m, Storable a)
326 => IOMode -> FilePath -> SerialT m (Array a) -> m ()
327fromChunksMode mode file xs = S.drain $
328 withFile file mode (\h -> S.mapM (FH.putChunk h) xs)
330-- | Write a stream of arrays to a file. Overwrites the file if it exists.
333{-# INLINE fromChunks #-}
334fromChunks :: (MonadAsync m, MonadCatch m, Storable a)
335 => FilePath -> SerialT m (Array a) -> m ()
336fromChunks = fromChunksMode WriteMode
338-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
341-- Note that if you use a chunk size less than 8K (GHC's default buffer
342-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
343-- do not want buffering to occur at GHC level as well. Same thing applies to
346-- | Like 'write' but provides control over the write buffer. Output will
347-- be written to the IO device as soon as we collect the specified number of
351{-# INLINE fromBytesWithBufferOf #-}
352fromBytesWithBufferOf :: (MonadAsync m, MonadCatch m)
353 => Int -> FilePath -> SerialT m Word8 -> m ()
354fromBytesWithBufferOf n file xs = fromChunks file $ AS.arraysOf n xs
356-- > write = 'writeWithBufferOf' defaultChunkSize
358-- | Write a byte stream to a file. Combines the bytes in chunks of size
359-- up to 'A.defaultChunkSize' before writing. If the file exists it is
360-- truncated to zero size before writing. If the file does not exist it is
361-- created. File is locked using single writer locking mode.
364{-# INLINE fromBytes #-}
365fromBytes :: (MonadAsync m, MonadCatch m) => FilePath -> SerialT m Word8 -> m ()
366fromBytes = fromBytesWithBufferOf defaultChunkSize
370write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
371write = toHandleWith A.defaultChunkSize
374-- | Write a stream of chunks to a handle. Each chunk in the stream is written
375-- to the device as a separate IO request.
378{-# INLINE writeChunks #-}
379writeChunks :: (MonadIO m, MonadCatch m, Storable a)
380 => FilePath -> Fold m (Array a) ()
381writeChunks path = Fold step initial extract
384 h <- liftIO (openFile path WriteMode)
385 fld <- FL.initialize (FH.writeChunks h)
386 `MC.onException` liftIO (hClose h)
387 return $ FL.Partial (fld, h)
389 r <- FL.snoc fld x `MC.onException` liftIO (hClose h)
390 return $ FL.Partial (r, h)
391 extract (Fold _ initial1 extract1, h) = do
395 FL.Partial fs -> extract1 fs
396 FL.Done fb -> return fb
398-- | @writeWithBufferOf chunkSize handle@ writes the input stream to @handle@.
399-- Bytes in the input stream are collected into a buffer until we have a chunk
400-- of size @chunkSize@ and then written to the IO device.
403{-# INLINE writeWithBufferOf #-}
404writeWithBufferOf :: (MonadIO m, MonadCatch m)
405 => Int -> FilePath -> Fold m Word8 ()
406writeWithBufferOf n path =
407 FL.chunksOf n (writeNUnsafe n) (writeChunks path)
409-- > write = 'writeWithBufferOf' A.defaultChunkSize
411-- | Write a byte stream to a file. Accumulates the input in chunks of up to
412-- 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize' before writing to
418write :: (MonadIO m, MonadCatch m) => FilePath -> Fold m Word8 ()
419write = writeWithBufferOf defaultChunkSize
421-- | Append a stream of arrays to a file.
424{-# INLINE appendChunks #-}
425appendChunks :: (MonadAsync m, MonadCatch m, Storable a)
426 => FilePath -> SerialT m (Array a) -> m ()
427appendChunks = fromChunksMode AppendMode
429-- | Like 'append' but provides control over the write buffer. Output will
430-- be written to the IO device as soon as we collect the specified number of
434{-# INLINE appendWithBufferOf #-}
435appendWithBufferOf :: (MonadAsync m, MonadCatch m)
436 => Int -> FilePath -> SerialT m Word8 -> m ()
437appendWithBufferOf n file xs = appendChunks file $ AS.arraysOf n xs
439-- | Append a byte stream to a file. Combines the bytes in chunks of size up to
440-- 'A.defaultChunkSize' before writing. If the file exists then the new data
441-- is appended to the file. If the file does not exist it is created. File is
442-- locked using single writer locking mode.
446append :: (MonadAsync m, MonadCatch m) => FilePath -> SerialT m Word8 -> m ()
447append = appendWithBufferOf defaultChunkSize
450-- | Like 'append' but the file is not locked for exclusive writes.
453{-# INLINE appendShared #-}
454appendShared :: MonadIO m => Handle -> SerialT m Word8 -> m ()
455appendShared = undefined
458-------------------------------------------------------------------------------
459-- IO with encoding/decoding Unicode characters
460-------------------------------------------------------------------------------
464-- > readUtf8 = decodeUtf8 . read
466-- Read a UTF8 encoded stream of unicode characters from a file handle.
469{-# INLINE readUtf8 #-}
470readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
471readUtf8 = decodeUtf8 . read
474-- > writeUtf8 h s = write h $ encodeUtf8 s
476-- Encode a stream of unicode characters to UTF8 and write it to the given file
477-- handle. Default block buffering applies to the writes.
480{-# INLINE writeUtf8 #-}
481writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
482writeUtf8 h s = write h $ encodeUtf8 s
484-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
485-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
486-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
487-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
491{-# INLINE writeUtf8ByLines #-}
492writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
493writeUtf8ByLines = undefined
495-- | Read UTF-8 lines from a file handle and apply the specified fold to each
496-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
499{-# INLINE readLines #-}
500readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
501readLines h f = foldLines (readUtf8 h) f
503-------------------------------------------------------------------------------
504-- Framing on a sequence
505-------------------------------------------------------------------------------
507-- | Read a stream from a file handle and split it into frames delimited by
508-- the specified sequence of elements. The supplied fold is applied on each
512{-# INLINE readFrames #-}
513readFrames :: (IsStream t, MonadIO m, Storable a)
514 => Array a -> Handle -> Fold m a b -> t m b
515readFrames = undefined -- foldFrames . read
517-- | Write a stream to the given file handle buffering up to frames separated
518-- by the given sequence or up to a maximum of @defaultChunkSize@.
521{-# INLINE writeByFrames #-}
522writeByFrames :: (IsStream t, MonadIO m, Storable a)
523 => Array a -> Handle -> t m a -> m ()
524writeByFrames = undefined
526-------------------------------------------------------------------------------
527-- Random Access IO (Seek)
528-------------------------------------------------------------------------------
530-- XXX handles could be shared, so we may not want to use the handle state at
531-- all for these APIs. we can use pread and pwrite instead. On windows we will
532-- need to use readFile/writeFile with an offset argument.
534-------------------------------------------------------------------------------
536-- | Read the element at the given index treating the file as an array.
539{-# INLINE readIndex #-}
540readIndex :: Storable a => Handle -> Int -> Maybe a
541readIndex arr i = undefined
543-- NOTE: To represent a range to read we have chosen (start, size) instead of
544-- (start, end). This helps in removing the ambiguity of whether "end" is
545-- included in the range or not.
547-- We could avoid specifying the range to be read and instead use "take size"
548-- on the stream, but it may end up reading more and then consume it partially.
550-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
551-- from @handle@ starting at the offset @pos@ from the beginning of the file.
553-- Reads are performed in chunks of size @chunkSize@. For block devices, to
554-- avoid reading partial blocks @chunkSize@ must align with the block size of
555-- the underlying device. If the underlying block size is unknown, it is a good
556-- idea to keep it a multiple 4KiB. This API ensures that the start of each
557-- chunk is aligned with @chunkSize@ from second chunk onwards.
559{-# INLINE readSliceWith #-}
560readSliceWith :: (IsStream t, MonadIO m, Storable a)
561 => Int -> Handle -> Int -> Int -> t m a
562readSliceWith chunkSize h pos len = undefined
564-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
565-- at index @i@ and reading up to @count@ elements in the forward direction
566-- ending at the index @i + count - 1@.
569{-# INLINE readSlice #-}
570readSlice :: (IsStream t, MonadIO m, Storable a)
571 => Handle -> Int -> Int -> t m a
572readSlice = readSliceWith defaultChunkSize
574-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
575-- at index @i@ and reading up to @count@ elements in the reverse direction
576-- ending at the index @i - count + 1@.
579{-# INLINE readSliceRev #-}
580readSliceRev :: (IsStream t, MonadIO m, Storable a)
581 => Handle -> Int -> Int -> t m a
582readSliceRev h i count = undefined
584-- | Write the given element at the given index in the file.
587{-# INLINE writeIndex #-}
588writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
589writeIndex h i a = undefined
591-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
592-- starting at index @i@ and writing up to @count@ elements in the forward
593-- direction ending at the index @i + count - 1@.
596{-# INLINE writeSlice #-}
597writeSlice :: (IsStream t, Monad m, Storable a)
598 => Handle -> Int -> Int -> t m a -> m ()
599writeSlice h i len s = undefined
601-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
602-- starting at index @i@ and writing up to @count@ elements in the reverse
603-- direction ending at the index @i - count + 1@.
606{-# INLINE writeSliceRev #-}
607writeSliceRev :: (IsStream t, Monad m, Storable a)
608 => Handle -> Int -> Int -> t m a -> m ()
609writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list