4-- Module : Streamly.Internal.FileSystem.File
5-- Copyright : (c) 2019 Composewell Technologies
8-- Maintainer : streamly@composewell.com
11-- Read and write streams and arrays to and from files specified by their paths
12-- in the file system. Unlike the handle based APIs which can have a read/write
13-- session consisting of multiple reads and writes to the handle, these APIs
14-- are one shot read or write APIs. These APIs open the file handle, perform
15-- the requested operation and close the handle. Thease are safer compared to
16-- the handle based APIs as there is no possibility of a file descriptor
19-- > import qualified Streamly.Internal.FileSystem.File as File
22module Streamly.Internal.FileSystem.File
25 -- | Stream data to or from a file or device sequentially. When reading,
26 -- the stream is lazy and generated on-demand as the consumer consumes it.
27 -- Read IO requests to the IO device are performed in chunks limited to a
28 -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
29 -- documentation. One IO request may or may not read the full
30 -- chunk. If the whole stream is not consumed, it is possible that we may
31 -- read slightly more from the IO device than what the consumer needed.
32 -- Unless specified otherwise in the API, writes are collected into chunks
33 -- of @defaultChunkSize@ before they are written to the IO device.
35 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
36 -- including disks, files, memory devices, terminals, pipes, sockets and
37 -- fifos. While random access APIs work only for files or devices that have
38 -- random access or seek capability for example disks, memory devices.
39 -- Devices like terminals, pipes, sockets and fifos do not have random
42 -- ** File IO Using Handle
58 , chunkReaderFromToWith
62 , putChunk -- writeChunk?
73 , fromBytes -- putBytes?
86 , readChunksWithBufferOf
87 , readChunksFromToWith
90 , toChunksWithBufferOf
92 , fromBytesWithBufferOf
96import Control.Monad.Catch (MonadCatch)
97import Control.Monad.IO.Class (MonadIO(..))
98import Data.Word (Word8)
99import System.IO (Handle, openFile, IOMode(..), hClose)
100import Prelude hiding (read)
102import qualified Control.Monad.Catch as MC
103import qualified System.IO as SIO
105import Streamly.Data.Fold (groupsOf, drain)
106import Streamly.Internal.Data.Array.Type (Array(..), pinnedWriteNUnsafe)
107import Streamly.Internal.Data.Fold.Type (Fold(..))
108import Streamly.Data.Stream (Stream)
109import Streamly.Internal.Data.Unbox (Unbox)
110import Streamly.Internal.Data.Unfold.Type (Unfold(..))
111-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
112import Streamly.Internal.System.IO (defaultChunkSize)
114import qualified Streamly.Data.Array as A
115import qualified Streamly.Data.Stream as S
116import qualified Streamly.Data.Unfold as UF
117import qualified Streamly.Internal.Data.Array.Type as IA (pinnedChunksOf)
118import qualified Streamly.Internal.Data.Unfold as UF (bracketIO)
119import qualified Streamly.Internal.Data.Fold.Type as FL
120 (Step(..), snoc, reduce)
121import qualified Streamly.Internal.FileSystem.Handle as FH
123-------------------------------------------------------------------------------
125-------------------------------------------------------------------------------
127-- The following references may be useful to build an understanding about the
130-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
131-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
132-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
133-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
134-- https://www.w3.org/TR/FileAPI/ for http file API.
136-------------------------------------------------------------------------------
138-------------------------------------------------------------------------------
140-- | @'withFile' name mode act@ opens a file using 'openFile' and passes
141-- the resulting handle to the computation @act@. The handle will be
142-- closed on exit from 'withFile', whether by normal termination or by
143-- raising an exception. If closing the handle raises an exception, then
144-- this exception will be raised by 'withFile' rather than any exception
149{-# INLINE withFile #-}
150withFile :: (MonadIO m, MonadCatch m)
151 => FilePath -> IOMode -> (Handle -> Stream m a) -> Stream m a
152withFile file mode = S.bracketIO (openFile file mode) hClose
154-- | Transform an 'Unfold' from a 'Handle' to an unfold from a 'FilePath'. The
155-- resulting unfold opens a handle in 'ReadMode', uses it using the supplied
156-- unfold and then makes sure that the handle is closed on normal termination
157-- or in case of an exception. If closing the handle raises an exception, then
158-- this exception will be raised by 'usingFile'.
162{-# INLINE usingFile #-}
163usingFile :: (MonadIO m, MonadCatch m)
164 => Unfold m Handle a -> Unfold m FilePath a
165usingFile = UF.bracketIO (`openFile` ReadMode) hClose
167{-# INLINE usingFile2 #-}
168usingFile2 :: (MonadIO m, MonadCatch m)
169 => Unfold m (x, Handle) a -> Unfold m (x, FilePath) a
170usingFile2 = UF.bracketIO before after
174 before (x, file) = do
175 h <- openFile file ReadMode
178 after (_, h) = hClose h
180{-# INLINE usingFile3 #-}
181usingFile3 :: (MonadIO m, MonadCatch m)
182 => Unfold m (x, y, z, Handle) a -> Unfold m (x, y, z, FilePath) a
183usingFile3 = UF.bracketIO before after
187 before (x, y, z, file) = do
188 h <- openFile file ReadMode
191 after (_, _, _, h) = hClose h
193-------------------------------------------------------------------------------
195-------------------------------------------------------------------------------
199-------------------------------------------------------------------------------
201-------------------------------------------------------------------------------
203-- | Write an array to a file. Overwrites the file if it exists.
207{-# INLINABLE putChunk #-}
208putChunk :: FilePath -> Array a -> IO ()
209putChunk file arr = SIO.withFile file WriteMode (`FH.putChunk` arr)
211-- | append an array to a file.
215{-# INLINABLE writeAppendArray #-}
216writeAppendArray :: FilePath -> Array a -> IO ()
217writeAppendArray file arr = SIO.withFile file AppendMode (`FH.putChunk` arr)
219-------------------------------------------------------------------------------
220-- Stream of Arrays IO
221-------------------------------------------------------------------------------
223-- | @readChunksWith size file@ reads a stream of arrays from file @file@.
224-- The maximum size of a single array is specified by @size@. The actual size
225-- read may be less than or equal to @size@.
229{-# INLINE readChunksWith #-}
230readChunksWith :: (MonadIO m, MonadCatch m)
231 => Int -> FilePath -> Stream m (Array Word8)
232readChunksWith size file =
233 withFile file ReadMode (FH.readChunksWith size)
235{-# DEPRECATED toChunksWithBufferOf "Please use 'readChunksWith' instead" #-}
236{-# INLINE toChunksWithBufferOf #-}
237toChunksWithBufferOf :: (MonadIO m, MonadCatch m)
238 => Int -> FilePath -> Stream m (Array Word8)
239toChunksWithBufferOf = readChunksWith
241-- XXX read 'Array a' instead of Word8
243-- | @readChunks file@ reads a stream of arrays from file @file@.
244-- The maximum size of a single array is limited to @defaultChunkSize@. The
245-- actual size read may be less than @defaultChunkSize@.
247-- > readChunks = readChunksWith defaultChunkSize
251{-# INLINE readChunks #-}
252readChunks :: (MonadIO m, MonadCatch m)
253 => FilePath -> Stream m (Array Word8)
254readChunks = readChunksWith defaultChunkSize
256{-# DEPRECATED toChunks "Please use 'readChunks' instead" #-}
257{-# INLINE toChunks #-}
258toChunks :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Array Word8)
261-------------------------------------------------------------------------------
262-- Read File to Stream
263-------------------------------------------------------------------------------
265-- TODO for concurrent streams implement readahead IO. We can send multiple
266-- read requests at the same time. For serial case we can use async IO. We can
267-- also control the read throughput in mbps or IOPS.
269-- | Unfold the tuple @(bufsize, filepath)@ into a stream of 'Word8' arrays.
270-- Read requests to the IO device are performed using a buffer of size
271-- @bufsize@. The size of an array in the resulting stream is always less than
272-- or equal to @bufsize@.
276{-# INLINE chunkReaderWith #-}
277chunkReaderWith :: (MonadIO m, MonadCatch m)
278 => Unfold m (Int, FilePath) (Array Word8)
279chunkReaderWith = usingFile2 FH.chunkReaderWith
281{-# DEPRECATED readChunksWithBufferOf
282 "Please use 'chunkReaderWith' instead" #-}
283{-# INLINE readChunksWithBufferOf #-}
284readChunksWithBufferOf :: (MonadIO m, MonadCatch m)
285 => Unfold m (Int, FilePath) (Array Word8)
286readChunksWithBufferOf = chunkReaderWith
288-- | Unfold the tuple @(from, to, bufsize, filepath)@ into a stream
290-- Read requests to the IO device are performed using a buffer of size
291-- @bufsize@ starting from absolute offset of @from@ till the absolute
292-- position of @to@. The size of an array in the resulting stream is always
293-- less than or equal to @bufsize@.
296{-# INLINE chunkReaderFromToWith #-}
297chunkReaderFromToWith :: (MonadIO m, MonadCatch m) =>
298 Unfold m (Int, Int, Int, FilePath) (Array Word8)
299chunkReaderFromToWith = usingFile3 FH.chunkReaderFromToWith
301{-# DEPRECATED readChunksFromToWith
302 "Please use 'chunkReaderFromToWith' instead" #-}
303{-# INLINE readChunksFromToWith #-}
304readChunksFromToWith :: (MonadIO m, MonadCatch m) =>
305 Unfold m (Int, Int, Int, FilePath) (Array Word8)
306readChunksFromToWith = chunkReaderFromToWith
308-- | Unfolds a 'FilePath' into a stream of 'Word8' arrays. Requests to the IO
309-- device are performed using a buffer of size
310-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize'. The
311-- size of arrays in the resulting stream are therefore less than or equal to
312-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize'.
315{-# INLINE chunkReader #-}
316chunkReader :: (MonadIO m, MonadCatch m) => Unfold m FilePath (Array Word8)
317chunkReader = usingFile FH.chunkReader
319-- | Unfolds the tuple @(bufsize, filepath)@ into a byte stream, read requests
320-- to the IO device are performed using buffers of @bufsize@.
323{-# INLINE readerWith #-}
324readerWith :: (MonadIO m, MonadCatch m) => Unfold m (Int, FilePath) Word8
325readerWith = usingFile2 FH.readerWith
327{-# DEPRECATED readWithBufferOf "Please use 'readerWith' instead" #-}
328{-# INLINE readWithBufferOf #-}
329readWithBufferOf :: (MonadIO m, MonadCatch m) =>
330 Unfold m (Int, FilePath) Word8
331readWithBufferOf = readerWith
333-- | Unfolds a file path into a byte stream. IO requests to the device are
334-- performed in sizes of
335-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize'.
339reader :: (MonadIO m, MonadCatch m) => Unfold m FilePath Word8
340reader = UF.many A.reader (usingFile FH.chunkReader)
342{-# INLINE concatChunks #-}
343concatChunks :: (Monad m, Unbox a) => Stream m (Array a) -> Stream m a
344concatChunks = S.unfoldMany A.reader
346-- | Generate a stream of bytes from a file specified by path. The stream ends
347-- when EOF is encountered. File is locked using multiple reader and single
348-- writer locking mode.
353read :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8
354read file = concatChunks $ withFile file ReadMode FH.readChunks
356{-# DEPRECATED toBytes "Please use 'read' instead" #-}
357{-# INLINE toBytes #-}
358toBytes :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8
362-- | Generate a stream of elements of the given type from a file 'Handle'. The
363-- stream ends when EOF is encountered. File is not locked for exclusive reads,
364-- writers can keep writing to the file.
367{-# INLINE readShared #-}
368readShared :: MonadIO m => Handle -> Stream m Word8
369readShared = undefined
372-------------------------------------------------------------------------------
374-------------------------------------------------------------------------------
376{-# INLINE fromChunksMode #-}
377fromChunksMode :: (MonadIO m, MonadCatch m)
378 => IOMode -> FilePath -> Stream m (Array a) -> m ()
379fromChunksMode mode file xs = S.fold drain $
380 withFile file mode (\h -> S.mapM (FH.putChunk h) xs)
382-- | Write a stream of arrays to a file. Overwrites the file if it exists.
386{-# INLINE fromChunks #-}
387fromChunks :: (MonadIO m, MonadCatch m)
388 => FilePath -> Stream m (Array a) -> m ()
389fromChunks = fromChunksMode WriteMode
391-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
394-- Note that if you use a chunk size less than 8K (GHC's default buffer
395-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
396-- do not want buffering to occur at GHC level as well. Same thing applies to
399-- | Like 'write' but provides control over the write buffer. Output will
400-- be written to the IO device as soon as we collect the specified number of
405{-# INLINE fromBytesWith #-}
406fromBytesWith :: (MonadIO m, MonadCatch m)
407 => Int -> FilePath -> Stream m Word8 -> m ()
408fromBytesWith n file xs = fromChunks file $ IA.pinnedChunksOf n xs
410{-# DEPRECATED fromBytesWithBufferOf "Please use 'fromBytesWith' instead" #-}
411{-# INLINE fromBytesWithBufferOf #-}
412fromBytesWithBufferOf :: (MonadIO m, MonadCatch m)
413 => Int -> FilePath -> Stream m Word8 -> m ()
414fromBytesWithBufferOf = fromBytesWith
416-- > write = 'writeWith' defaultChunkSize
418-- | Write a byte stream to a file. Combines the bytes in chunks of size
419-- up to 'A.defaultChunkSize' before writing. If the file exists it is
420-- truncated to zero size before writing. If the file does not exist it is
421-- created. File is locked using single writer locking mode.
424{-# INLINE fromBytes #-}
425fromBytes :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8 -> m ()
426fromBytes = fromBytesWith defaultChunkSize
430write :: (MonadIO m, Storable a) => Handle -> Stream m a -> m ()
431write = toHandleWith A.defaultChunkSize
434-- | Write a stream of chunks to a handle. Each chunk in the stream is written
435-- to the device as a separate IO request.
438{-# INLINE writeChunks #-}
439writeChunks :: (MonadIO m, MonadCatch m)
440 => FilePath -> Fold m (Array a) ()
441writeChunks path = Fold step initial extract final
444 h <- liftIO (openFile path WriteMode)
445 fld <- FL.reduce (FH.writeChunks h)
446 `MC.onException` liftIO (hClose h)
447 return $ FL.Partial (fld, h)
449 r <- FL.snoc fld x `MC.onException` liftIO (hClose h)
450 return $ FL.Partial (r, h)
452 extract _ = return ()
454 final (Fold _ initial1 _ final1, h) = do
458 FL.Partial fs -> final1 fs
459 FL.Done () -> return ()
461-- | @writeWith chunkSize handle@ writes the input stream to @handle@.
462-- Bytes in the input stream are collected into a buffer until we have a chunk
463-- of size @chunkSize@ and then written to the IO device.
466{-# INLINE writeWith #-}
467writeWith :: (MonadIO m, MonadCatch m)
468 => Int -> FilePath -> Fold m Word8 ()
470 groupsOf n (pinnedWriteNUnsafe n) (writeChunks path)
472{-# DEPRECATED writeWithBufferOf "Please use 'writeWith' instead" #-}
473{-# INLINE writeWithBufferOf #-}
474writeWithBufferOf :: (MonadIO m, MonadCatch m)
475 => Int -> FilePath -> Fold m Word8 ()
476writeWithBufferOf = writeWith
478-- > write = 'writeWith' A.defaultChunkSize
480-- | Write a byte stream to a file. Accumulates the input in chunks of up to
481-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize' before writing to
487write :: (MonadIO m, MonadCatch m) => FilePath -> Fold m Word8 ()
488write = writeWith defaultChunkSize
490-- | Append a stream of arrays to a file.
494{-# INLINE writeAppendChunks #-}
495writeAppendChunks :: (MonadIO m, MonadCatch m)
496 => FilePath -> Stream m (Array a) -> m ()
497writeAppendChunks = fromChunksMode AppendMode
499-- | Like 'append' but provides control over the write buffer. Output will
500-- be written to the IO device as soon as we collect the specified number of
505{-# INLINE writeAppendWith #-}
506writeAppendWith :: (MonadIO m, MonadCatch m)
507 => Int -> FilePath -> Stream m Word8 -> m ()
508writeAppendWith n file xs =
509 writeAppendChunks file $ IA.pinnedChunksOf n xs
511-- | Append a byte stream to a file. Combines the bytes in chunks of size up to
512-- 'A.defaultChunkSize' before writing. If the file exists then the new data
513-- is appended to the file. If the file does not exist it is created. File is
514-- locked using single writer locking mode.
518{-# INLINE writeAppend #-}
519writeAppend :: (MonadIO m, MonadCatch m) => FilePath -> Stream m Word8 -> m ()
520writeAppend = writeAppendWith defaultChunkSize
523-- | Like 'append' but the file is not locked for exclusive writes.
526{-# INLINE appendShared #-}
527appendShared :: MonadIO m => Handle -> Stream m Word8 -> m ()
528appendShared = undefined
531-------------------------------------------------------------------------------
532-- IO with encoding/decoding Unicode characters
533-------------------------------------------------------------------------------
537-- > readUtf8 = decodeUtf8 . read
539-- Read a UTF8 encoded stream of unicode characters from a file handle.
542{-# INLINE readUtf8 #-}
543readUtf8 :: MonadIO m => Handle -> Stream m Char
544readUtf8 = decodeUtf8 . read
547-- > writeUtf8 h s = write h $ encodeUtf8 s
549-- Encode a stream of unicode characters to UTF8 and write it to the given file
550-- handle. Default block buffering applies to the writes.
553{-# INLINE writeUtf8 #-}
554writeUtf8 :: MonadIO m => Handle -> Stream m Char -> m ()
555writeUtf8 h s = write h $ encodeUtf8 s
557-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
558-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
559-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
560-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
564{-# INLINE writeUtf8ByLines #-}
565writeUtf8ByLines :: MonadIO m => Handle -> Stream m Char -> m ()
566writeUtf8ByLines = undefined
568-- | Read UTF-8 lines from a file handle and apply the specified fold to each
569-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
572{-# INLINE readLines #-}
573readLines :: MonadIO m => Handle -> Fold m Char b -> Stream m b
574readLines h f = foldLines (readUtf8 h) f
576-------------------------------------------------------------------------------
577-- Framing on a sequence
578-------------------------------------------------------------------------------
580-- | Read a stream from a file handle and split it into frames delimited by
581-- the specified sequence of elements. The supplied fold is applied on each
585{-# INLINE readFrames #-}
586readFrames :: (MonadIO m, Storable a)
587 => Array a -> Handle -> Fold m a b -> Stream m b
588readFrames = undefined -- foldFrames . read
590-- | Write a stream to the given file handle buffering up to frames separated
591-- by the given sequence or up to a maximum of @defaultChunkSize@.
594{-# INLINE writeByFrames #-}
595writeByFrames :: (MonadIO m, Storable a)
596 => Array a -> Handle -> Stream m a -> m ()
597writeByFrames = undefined
599-------------------------------------------------------------------------------
600-- Random Access IO (Seek)
601-------------------------------------------------------------------------------
603-- XXX handles could be shared, so we may not want to use the handle state at
604-- all for these APIs. we can use pread and pwrite instead. On windows we will
605-- need to use readFile/writeFile with an offset argument.
607-------------------------------------------------------------------------------
609-- | Read the element at the given index treating the file as an array.
612{-# INLINE readIndex #-}
613readIndex :: Storable a => Handle -> Int -> Maybe a
614readIndex arr i = undefined
616-- NOTE: To represent a range to read we have chosen (start, size) instead of
617-- (start, end). This helps in removing the ambiguity of whether "end" is
618-- included in the range or not.
620-- We could avoid specifying the range to be read and instead use "take size"
621-- on the stream, but it may end up reading more and then consume it partially.
623-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
624-- from @handle@ starting at the offset @pos@ from the beginning of the file.
626-- Reads are performed in chunks of size @chunkSize@. For block devices, to
627-- avoid reading partial blocks @chunkSize@ must align with the block size of
628-- the underlying device. If the underlying block size is unknown, it is a good
629-- idea to keep it a multiple 4KiB. This API ensures that the start of each
630-- chunk is aligned with @chunkSize@ from second chunk onwards.
632{-# INLINE readSliceWith #-}
633readSliceWith :: (MonadIO m, Storable a)
634 => Int -> Handle -> Int -> Int -> Stream m a
635readSliceWith chunkSize h pos len = undefined
637-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
638-- at index @i@ and reading up to @count@ elements in the forward direction
639-- ending at the index @i + count - 1@.
642{-# INLINE readSlice #-}
643readSlice :: (MonadIO m, Storable a)
644 => Handle -> Int -> Int -> Stream m a
645readSlice = readSliceWith defaultChunkSize
647-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
648-- at index @i@ and reading up to @count@ elements in the reverse direction
649-- ending at the index @i - count + 1@.
652{-# INLINE readSliceRev #-}
653readSliceRev :: (MonadIO m, Storable a)
654 => Handle -> Int -> Int -> Stream m a
655readSliceRev h i count = undefined
657-- | Write the given element at the given index in the file.
660{-# INLINE writeIndex #-}
661writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
662writeIndex h i a = undefined
664-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
665-- starting at index @i@ and writing up to @count@ elements in the forward
666-- direction ending at the index @i + count - 1@.
669{-# INLINE writeSlice #-}
670writeSlice :: (Monad m, Storable a)
671 => Handle -> Int -> Int -> Stream m a -> m ()
672writeSlice h i len s = undefined
674-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
675-- starting at index @i@ and writing up to @count@ elements in the reverse
676-- direction ending at the index @i - count + 1@.
679{-# INLINE writeSliceRev #-}
680writeSliceRev :: (Monad m, Storable a)
681 => Handle -> Int -> Int -> Stream m a -> m ()
682writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list