2{-# LANGUAGE BangPatterns #-}
3{-# LANGUAGE FlexibleContexts #-}
4{-# LANGUAGE MagicHash #-}
5{-# LANGUAGE RecordWildCards #-}
6{-# LANGUAGE UnboxedTuples #-}
11-- Module : Streamly.Internal.FileSystem.File
12-- Copyright : (c) 2019 Harendra Kumar
15-- Maintainer : streamly@composewell.com
16-- Stability : experimental
19-- Read and write streams and arrays to and from files specified by their paths
20-- in the file system. Unlike the handle based APIs which can have a read/write
21-- session consisting of multiple reads and writes to the handle, these APIs
22-- are one shot read or write APIs. These APIs open the file handle, perform
23-- the requested operation and close the handle. Thease are safer compared to
24-- the handle based APIs as there is no possibility of a file descriptor
27-- > import qualified Streamly.Internal.FileSystem.File as File
30module Streamly.Internal.FileSystem.File
33 -- | Stream data to or from a file or device sequentially. When reading,
34 -- the stream is lazy and generated on-demand as the consumer consumes it.
35 -- Read IO requests to the IO device are performed in chunks limited to a
36 -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
37 -- documentation. One IO request may or may not read the full
38 -- chunk. If the whole stream is not consumed, it is possible that we may
39 -- read slightly more from the IO device than what the consumer needed.
40 -- Unless specified otherwise in the API, writes are collected into chunks
41 -- of @defaultChunkSize@ before they are written to the IO device.
43 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
44 -- including disks, files, memory devices, terminals, pipes, sockets and
45 -- fifos. While random access APIs work only for files or devices that have
46 -- random access or seek capability for example disks, memory devices.
47 -- Devices like terminals, pipes, sockets and fifos do not have random
50 -- ** File IO Using Handle
68 , toChunksWithBufferOf
79 , fromBytesWithBufferOf
95import Control.Monad.Catch (MonadCatch)
96import Control.Monad.IO.Class (MonadIO(..))
97import Data.Word (Word8)
98import Foreign.Storable (Storable(..))
99import System.IO (Handle, openFile, IOMode(..), hClose)
100import Prelude hiding (read)
102import qualified Control.Monad.Catch as MC
103import qualified System.IO as SIO
105import Streamly.Internal.Data.Fold.Types (Fold(..))
106import Streamly.Internal.Data.Unfold.Types (Unfold(..))
107import Streamly.Internal.Memory.Array.Types
108 (Array(..), defaultChunkSize, writeNUnsafe)
109import Streamly.Internal.Data.Stream.Serial (SerialT)
110import Streamly.Internal.Data.Stream.StreamK.Type (IsStream)
111import Streamly.Internal.Data.SVar (MonadAsync)
112-- import Streamly.Data.Fold (Fold)
113-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
115import qualified Streamly.Internal.Data.Fold.Types as FL
116import qualified Streamly.Internal.Data.Unfold as UF
117import qualified Streamly.Internal.FileSystem.Handle as FH
118import qualified Streamly.Internal.Memory.ArrayStream as AS
119import qualified Streamly.Memory.Array as A
120import qualified Streamly.Prelude as S
122-------------------------------------------------------------------------------
124-------------------------------------------------------------------------------
126-- The following references may be useful to build an understanding about the
129-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
130-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
131-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
132-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
133-- https://www.w3.org/TR/FileAPI/ for http file API.
135-------------------------------------------------------------------------------
137-------------------------------------------------------------------------------
139-- | @'withFile' name mode act@ opens a file using 'openFile' and passes
140-- the resulting handle to the computation @act@. The handle will be
141-- closed on exit from 'withFile', whether by normal termination or by
142-- raising an exception. If closing the handle raises an exception, then
143-- this exception will be raised by 'withFile' rather than any exception
148{-# INLINE withFile #-}
149withFile :: (IsStream t, MonadCatch m, MonadIO m)
150 => FilePath -> IOMode -> (Handle -> t m a) -> t m a
151withFile file mode = S.bracket (liftIO $ openFile file mode) (liftIO . hClose)
153-- | Transform an 'Unfold' from a 'Handle' to an unfold from a 'FilePath'. The
154-- resulting unfold opens a handle in 'ReadMode', uses it using the supplied
155-- unfold and then makes sure that the handle is closed on normal termination
156-- or in case of an exception. If closing the handle raises an exception, then
157-- this exception will be raised by 'usingFile'.
161{-# INLINABLE usingFile #-}
162usingFile :: (MonadCatch m, MonadIO m)
163 => Unfold m Handle a -> Unfold m FilePath a
165 UF.bracket (\file -> liftIO $ openFile file ReadMode)
168-------------------------------------------------------------------------------
170-------------------------------------------------------------------------------
174-------------------------------------------------------------------------------
176-------------------------------------------------------------------------------
178-- | Write an array to a file. Overwrites the file if it exists.
181{-# INLINABLE writeArray #-}
182writeArray :: Storable a => FilePath -> Array a -> IO ()
183writeArray file arr = SIO.withFile file WriteMode (\h -> FH.writeArray h arr)
185-- | append an array to a file.
188{-# INLINABLE appendArray #-}
189appendArray :: Storable a => FilePath -> Array a -> IO ()
190appendArray file arr = SIO.withFile file AppendMode (\h -> FH.writeArray h arr)
192-------------------------------------------------------------------------------
193-- Stream of Arrays IO
194-------------------------------------------------------------------------------
196-- | @toChunksWithBufferOf size file@ reads a stream of arrays from file @file@.
197-- The maximum size of a single array is specified by @size@. The actual size
198-- read may be less than or equal to @size@.
199{-# INLINABLE toChunksWithBufferOf #-}
200toChunksWithBufferOf :: (IsStream t, MonadCatch m, MonadIO m)
201 => Int -> FilePath -> t m (Array Word8)
202toChunksWithBufferOf size file =
203 withFile file ReadMode (FH.toChunksWithBufferOf size)
205-- XXX read 'Array a' instead of Word8
207-- | @toChunks file@ reads a stream of arrays from file @file@.
208-- The maximum size of a single array is limited to @defaultChunkSize@. The
209-- actual size read may be less than @defaultChunkSize@.
211-- > toChunks = toChunksWithBufferOf defaultChunkSize
214{-# INLINE toChunks #-}
215toChunks :: (IsStream t, MonadCatch m, MonadIO m)
216 => FilePath -> t m (Array Word8)
217toChunks = toChunksWithBufferOf defaultChunkSize
219-------------------------------------------------------------------------------
220-- Read File to Stream
221-------------------------------------------------------------------------------
223-- TODO for concurrent streams implement readahead IO. We can send multiple
224-- read requests at the same time. For serial case we can use async IO. We can
225-- also control the read throughput in mbps or IOPS.
228-- | Unfolds the tuple @(bufsize, filepath)@ into a byte stream, read requests
229-- to the IO device are performed using buffers of @bufsize@.
232{-# INLINE readWithBufferOf #-}
233readWithBufferOf :: MonadIO m => Unfold m (Int, FilePath) Word8
234readWithBufferOf = UF.concat (usingFilexxx FH.readChunksWithBufferOf) A.read
237-- | Unfolds a file path into a byte stream. IO requests to the device are
238-- performed in sizes of
239-- 'Streamly.Internal.Memory.Array.Types.defaultChunkSize'.
243read :: (MonadCatch m, MonadIO m) => Unfold m FilePath Word8
244read = UF.concat (usingFile FH.readChunks) A.read
247-- | @readInChunksOf chunkSize handle@ reads a byte stream from a file
248-- handle, reads are performed in chunks of up to @chunkSize@. The stream ends
249-- as soon as EOF is encountered.
251{-# INLINE readInChunksOf #-}
252readInChunksOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
253readInChunksOf chunkSize h = A.flattenArrays $ toChunksWithBufferOf chunkSize h
257-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
259-- > read = 'readByChunks' defaultChunkSize
260-- | Generate a stream of bytes from a file specified by path. The stream ends
261-- when EOF is encountered. File is locked using multiple reader and single
262-- writer locking mode.
266{-# INLINE toBytes #-}
267toBytes :: (IsStream t, MonadCatch m, MonadIO m) => FilePath -> t m Word8
268toBytes file = AS.concat $ withFile file ReadMode FH.toChunks
271-- | Generate a stream of elements of the given type from a file 'Handle'. The
272-- stream ends when EOF is encountered. File is not locked for exclusive reads,
273-- writers can keep writing to the file.
276{-# INLINE readShared #-}
277readShared :: (IsStream t, MonadIO m) => Handle -> t m Word8
278readShared = undefined
280-- | Read a stream from a given file path. When end of file (EOF) is reached
281-- this API waits for more data to be written to the file and keeps reading it
285{-# INLINE readTailForever #-}
286readTailForever :: (IsStream t, MonadIO m) => Handle -> t m Word8
287readTailForever = undefined
290-------------------------------------------------------------------------------
292-------------------------------------------------------------------------------
294{-# INLINE fromChunksMode #-}
295fromChunksMode :: (MonadAsync m, MonadCatch m, Storable a)
296 => IOMode -> FilePath -> SerialT m (Array a) -> m ()
297fromChunksMode mode file xs = S.drain $
298 withFile file mode (\h -> S.mapM (liftIO . FH.writeArray h) xs)
300-- | Write a stream of arrays to a file. Overwrites the file if it exists.
303{-# INLINE fromChunks #-}
304fromChunks :: (MonadAsync m, MonadCatch m, Storable a)
305 => FilePath -> SerialT m (Array a) -> m ()
306fromChunks = fromChunksMode WriteMode
308-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
311-- Note that if you use a chunk size less than 8K (GHC's default buffer
312-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
313-- do not want buffering to occur at GHC level as well. Same thing applies to
316-- | Like 'write' but provides control over the write buffer. Output will
317-- be written to the IO device as soon as we collect the specified number of
321{-# INLINE fromBytesWithBufferOf #-}
322fromBytesWithBufferOf :: (MonadAsync m, MonadCatch m)
323 => Int -> FilePath -> SerialT m Word8 -> m ()
324fromBytesWithBufferOf n file xs = fromChunks file $ AS.arraysOf n xs
326-- > write = 'writeWithBufferOf' defaultChunkSize
328-- | Write a byte stream to a file. Combines the bytes in chunks of size
329-- up to 'A.defaultChunkSize' before writing. If the file exists it is
330-- truncated to zero size before writing. If the file does not exist it is
331-- created. File is locked using single writer locking mode.
334{-# INLINE fromBytes #-}
335fromBytes :: (MonadAsync m, MonadCatch m) => FilePath -> SerialT m Word8 -> m ()
336fromBytes = fromBytesWithBufferOf defaultChunkSize
340write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
341write = toHandleWith A.defaultChunkSize
344-- | Write a stream of chunks to a handle. Each chunk in the stream is written
345-- to the device as a separate IO request.
348{-# INLINE writeChunks #-}
349writeChunks :: (MonadIO m, MonadCatch m, Storable a)
350 => FilePath -> Fold m (Array a) ()
351writeChunks path = Fold step initial extract
354 h <- liftIO (openFile path WriteMode)
355 fld <- FL.initialize (FH.writeChunks h)
356 `MC.onException` (liftIO $ hClose h)
359 r <- FL.runStep fld x `MC.onException` (liftIO $ hClose h)
361 extract ((Fold _ initial1 extract1), h) = do
363 initial1 >>= extract1
365-- | @writeWithBufferOf chunkSize handle@ writes the input stream to @handle@.
366-- Bytes in the input stream are collected into a buffer until we have a chunk
367-- of size @chunkSize@ and then written to the IO device.
370{-# INLINE writeWithBufferOf #-}
371writeWithBufferOf :: (MonadIO m, MonadCatch m)
372 => Int -> FilePath -> Fold m Word8 ()
373writeWithBufferOf n path =
374 FL.lchunksOf n (writeNUnsafe n) (writeChunks path)
376-- > write = 'writeWithBufferOf' A.defaultChunkSize
378-- | Write a byte stream to a file. Accumulates the input in chunks of up to
379-- 'Streamly.Internal.Memory.Array.Types.defaultChunkSize' before writing to
385write :: (MonadIO m, MonadCatch m) => FilePath -> Fold m Word8 ()
386write = writeWithBufferOf defaultChunkSize
388-- | Append a stream of arrays to a file.
391{-# INLINE appendChunks #-}
392appendChunks :: (MonadAsync m, MonadCatch m, Storable a)
393 => FilePath -> SerialT m (Array a) -> m ()
394appendChunks = fromChunksMode AppendMode
396-- | Like 'append' but provides control over the write buffer. Output will
397-- be written to the IO device as soon as we collect the specified number of
401{-# INLINE appendWithBufferOf #-}
402appendWithBufferOf :: (MonadAsync m, MonadCatch m)
403 => Int -> FilePath -> SerialT m Word8 -> m ()
404appendWithBufferOf n file xs = appendChunks file $ AS.arraysOf n xs
406-- | Append a byte stream to a file. Combines the bytes in chunks of size up to
407-- 'A.defaultChunkSize' before writing. If the file exists then the new data
408-- is appended to the file. If the file does not exist it is created. File is
409-- locked using single writer locking mode.
413append :: (MonadAsync m, MonadCatch m) => FilePath -> SerialT m Word8 -> m ()
414append = appendWithBufferOf defaultChunkSize
417-- | Like 'append' but the file is not locked for exclusive writes.
420{-# INLINE appendShared #-}
421appendShared :: MonadIO m => Handle -> SerialT m Word8 -> m ()
422appendShared = undefined
425-------------------------------------------------------------------------------
426-- IO with encoding/decoding Unicode characters
427-------------------------------------------------------------------------------
431-- > readUtf8 = decodeUtf8 . read
433-- Read a UTF8 encoded stream of unicode characters from a file handle.
436{-# INLINE readUtf8 #-}
437readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
438readUtf8 = decodeUtf8 . read
441-- > writeUtf8 h s = write h $ encodeUtf8 s
443-- Encode a stream of unicode characters to UTF8 and write it to the given file
444-- handle. Default block buffering applies to the writes.
447{-# INLINE writeUtf8 #-}
448writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
449writeUtf8 h s = write h $ encodeUtf8 s
451-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
452-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
453-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
454-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
458{-# INLINE writeUtf8ByLines #-}
459writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
460writeUtf8ByLines = undefined
462-- | Read UTF-8 lines from a file handle and apply the specified fold to each
463-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
466{-# INLINE readLines #-}
467readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
468readLines h f = foldLines (readUtf8 h) f
470-------------------------------------------------------------------------------
471-- Framing on a sequence
472-------------------------------------------------------------------------------
474-- | Read a stream from a file handle and split it into frames delimited by
475-- the specified sequence of elements. The supplied fold is applied on each
479{-# INLINE readFrames #-}
480readFrames :: (IsStream t, MonadIO m, Storable a)
481 => Array a -> Handle -> Fold m a b -> t m b
482readFrames = undefined -- foldFrames . read
484-- | Write a stream to the given file handle buffering up to frames separated
485-- by the given sequence or up to a maximum of @defaultChunkSize@.
488{-# INLINE writeByFrames #-}
489writeByFrames :: (IsStream t, MonadIO m, Storable a)
490 => Array a -> Handle -> t m a -> m ()
491writeByFrames = undefined
493-------------------------------------------------------------------------------
494-- Random Access IO (Seek)
495-------------------------------------------------------------------------------
497-- XXX handles could be shared, so we may not want to use the handle state at
498-- all for these APIs. we can use pread and pwrite instead. On windows we will
499-- need to use readFile/writeFile with an offset argument.
501-------------------------------------------------------------------------------
503-- | Read the element at the given index treating the file as an array.
506{-# INLINE readIndex #-}
507readIndex :: Storable a => Handle -> Int -> Maybe a
508readIndex arr i = undefined
510-- NOTE: To represent a range to read we have chosen (start, size) instead of
511-- (start, end). This helps in removing the ambiguity of whether "end" is
512-- included in the range or not.
514-- We could avoid specifying the range to be read and instead use "take size"
515-- on the stream, but it may end up reading more and then consume it partially.
517-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
518-- from @handle@ starting at the offset @pos@ from the beginning of the file.
520-- Reads are performed in chunks of size @chunkSize@. For block devices, to
521-- avoid reading partial blocks @chunkSize@ must align with the block size of
522-- the underlying device. If the underlying block size is unknown, it is a good
523-- idea to keep it a multiple 4KiB. This API ensures that the start of each
524-- chunk is aligned with @chunkSize@ from second chunk onwards.
526{-# INLINE readSliceWith #-}
527readSliceWith :: (IsStream t, MonadIO m, Storable a)
528 => Int -> Handle -> Int -> Int -> t m a
529readSliceWith chunkSize h pos len = undefined
531-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
532-- at index @i@ and reading up to @count@ elements in the forward direction
533-- ending at the index @i + count - 1@.
536{-# INLINE readSlice #-}
537readSlice :: (IsStream t, MonadIO m, Storable a)
538 => Handle -> Int -> Int -> t m a
539readSlice = readSliceWith defaultChunkSize
541-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
542-- at index @i@ and reading up to @count@ elements in the reverse direction
543-- ending at the index @i - count + 1@.
546{-# INLINE readSliceRev #-}
547readSliceRev :: (IsStream t, MonadIO m, Storable a)
548 => Handle -> Int -> Int -> t m a
549readSliceRev h i count = undefined
551-- | Write the given element at the given index in the file.
554{-# INLINE writeIndex #-}
555writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
556writeIndex h i a = undefined
558-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
559-- starting at index @i@ and writing up to @count@ elements in the forward
560-- direction ending at the index @i + count - 1@.
563{-# INLINE writeSlice #-}
564writeSlice :: (IsStream t, Monad m, Storable a)
565 => Handle -> Int -> Int -> t m a -> m ()
566writeSlice h i len s = undefined
568-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
569-- starting at index @i@ and writing up to @count@ elements in the reverse
570-- direction ending at the index @i - count + 1@.
573{-# INLINE writeSliceRev #-}
574writeSliceRev :: (IsStream t, Monad m, Storable a)
575 => Handle -> Int -> Int -> t m a -> m ()
576writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list