1{-# OPTIONS_HADDOCK hide #-}
3{-# LANGUAGE BangPatterns #-}
4{-# LANGUAGE FlexibleContexts #-}
5{-# LANGUAGE MagicHash #-}
6{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE UnboxedTuples #-}
12-- Module : Streamly.Internal.FileSystem.File
13-- Copyright : (c) 2019 Harendra Kumar
16-- Maintainer : streamly@composewell.com
17-- Stability : experimental
20-- Read and write streams and arrays to and from files specified by their paths
21-- in the file system. Unlike the handle based APIs which can have a read/write
22-- session consisting of multiple reads and writes to the handle, these APIs
23-- are one shot read or write APIs. These APIs open the file handle, perform
24-- the requested operation and close the handle. Thease are safer compared to
25-- the handle based APIs as there is no possiblity of a file descriptor
28-- > import qualified Streamly.Internal.FileSystem.File as File
31module Streamly.Internal.FileSystem.File
34 -- | Stream data to or from a file or device sequentially. When reading,
35 -- the stream is lazy and generated on-demand as the consumer consumes it.
36 -- Read IO requests to the IO device are performed in chunks limited to a
37 -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
38 -- documentation. One IO request may or may not read the full
39 -- chunk. If the whole stream is not consumed, it is possible that we may
40 -- read slightly more from the IO device than what the consumer needed.
41 -- Unless specified otherwise in the API, writes are collected into chunks
42 -- of @defaultChunkSize@ before they are written to the IO device.
44 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
45 -- including disks, files, memory devices, terminals, pipes, sockets and
46 -- fifos. While random access APIs work only for files or devices that have
47 -- random access or seek capability for example disks, memory devices.
48 -- Devices like terminals, pipes, sockets and fifos do not have random
51 -- ** File IO Using Handle
67 , toChunksWithBufferOf
68 -- , toChunksWithBufferOf
91import Control.Monad.Catch (MonadCatch)
92import Control.Monad.IO.Class (MonadIO(..))
93import Data.Word (Word8)
94import Foreign.Storable (Storable(..))
95import System.IO (Handle, openFile, IOMode(..), hClose)
96import Prelude hiding (read)
98import qualified System.IO as SIO
100import Streamly.Internal.Memory.Array.Types (Array(..), defaultChunkSize)
101import Streamly.Streams.Serial (SerialT)
102import Streamly.Streams.StreamK.Type (IsStream)
103import Streamly.Internal.Data.SVar (MonadAsync)
104-- import Streamly.Data.Fold (Fold)
105-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
107import qualified Streamly.Internal.FileSystem.Handle as FH
108import qualified Streamly.Internal.Memory.ArrayStream as AS
109import qualified Streamly.Prelude as S
111-------------------------------------------------------------------------------
113-------------------------------------------------------------------------------
115-- The following references may be useful to build an understanding about the
118-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
119-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
120-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
121-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
122-- https://www.w3.org/TR/FileAPI/ for http file API.
124-------------------------------------------------------------------------------
126-------------------------------------------------------------------------------
128-- | @'withFile' name mode act@ opens a file using 'openFile' and passes
129-- the resulting handle to the computation @act@. The handle will be
130-- closed on exit from 'withFile', whether by normal termination or by
131-- raising an exception. If closing the handle raises an exception, then
132-- this exception will be raised by 'withFile' rather than any exception
134{-# INLINE withFile #-}
135withFile :: (IsStream t, MonadCatch m, MonadIO m)
136 => FilePath -> IOMode -> (Handle -> t m a) -> t m a
137withFile file mode = S.bracket (liftIO $ openFile file mode) (liftIO . hClose)
139-------------------------------------------------------------------------------
141-------------------------------------------------------------------------------
145-------------------------------------------------------------------------------
147-------------------------------------------------------------------------------
149-- | Write an array to a file. Overwrites the file if it exists.
152{-# INLINABLE writeArray #-}
153writeArray :: Storable a => FilePath -> Array a -> IO ()
154writeArray file arr = SIO.withFile file WriteMode (\h -> FH.writeArray h arr)
156-- | append an array to a file.
159{-# INLINABLE appendArray #-}
160appendArray :: Storable a => FilePath -> Array a -> IO ()
161appendArray file arr = SIO.withFile file AppendMode (\h -> FH.writeArray h arr)
163-------------------------------------------------------------------------------
164-- Stream of Arrays IO
165-------------------------------------------------------------------------------
167-- | @toChunksWithBufferOf size file@ reads a stream of arrays from file @file@.
168-- The maximum size of a single array is specified by @size@. The actual size
169-- read may be less than or equal to @size@.
170{-# INLINABLE toChunksWithBufferOf #-}
171toChunksWithBufferOf :: (IsStream t, MonadCatch m, MonadIO m)
172 => Int -> FilePath -> t m (Array Word8)
173toChunksWithBufferOf size file =
174 withFile file ReadMode (FH.toChunksWithBufferOf size)
176-- XXX read 'Array a' instead of Word8
178-- | @toChunks file@ reads a stream of arrays from file @file@.
179-- The maximum size of a single array is limited to @defaultChunkSize@. The
180-- actual size read may be less than @defaultChunkSize@.
182-- > toChunks = toChunksWithBufferOf defaultChunkSize
185{-# INLINE toChunks #-}
186toChunks :: (IsStream t, MonadCatch m, MonadIO m)
187 => FilePath -> t m (Array Word8)
188toChunks = toChunksWithBufferOf defaultChunkSize
190-------------------------------------------------------------------------------
191-- Read File to Stream
192-------------------------------------------------------------------------------
194-- TODO for concurrent streams implement readahead IO. We can send multiple
195-- read requests at the same time. For serial case we can use async IO. We can
196-- also control the read throughput in mbps or IOPS.
199-- | @readInChunksOf chunkSize handle@ reads a byte stream from a file
200-- handle, reads are performed in chunks of up to @chunkSize@. The stream ends
201-- as soon as EOF is encountered.
203{-# INLINE readInChunksOf #-}
204readInChunksOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
205readInChunksOf chunkSize h = A.flattenArrays $ toChunksWithBufferOf chunkSize h
209-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
211-- > read = 'readByChunks' defaultChunkSize
212-- | Generate a stream of elements of the given type from a file specified by
213-- path. The stream ends when EOF is encountered. File is locked using multiple
214-- reader and single writer locking mode.
218read :: (IsStream t, MonadCatch m, MonadIO m) => FilePath -> t m Word8
219read file = AS.concat $ withFile file ReadMode FH.toChunks
222-- | Generate a stream of elements of the given type from a file 'Handle'. The
223-- stream ends when EOF is encountered. File is not locked for exclusive reads,
224-- writers can keep writing to the file.
227{-# INLINE readShared #-}
228readShared :: (IsStream t, MonadIO m) => Handle -> t m Word8
229readShared = undefined
231-- | Read a stream from a given file path. When end of file (EOF) is reached
232-- this API waits for more data to be written to the file and keeps reading it
236{-# INLINE readTailForever #-}
237readTailForever :: (IsStream t, MonadIO m) => Handle -> t m Word8
238readTailForever = undefined
241-------------------------------------------------------------------------------
243-------------------------------------------------------------------------------
245{-# INLINE writeChunksMode #-}
246writeChunksMode :: (MonadAsync m, MonadCatch m, Storable a)
247 => IOMode -> FilePath -> SerialT m (Array a) -> m ()
248writeChunksMode mode file xs = S.drain $
249 withFile file mode (\h -> S.mapM (liftIO . FH.writeArray h) xs)
251-- | Write a stream of arrays to a file. Overwrites the file if it exists.
254{-# INLINE writeChunks #-}
255writeChunks :: (MonadAsync m, MonadCatch m, Storable a)
256 => FilePath -> SerialT m (Array a) -> m ()
257writeChunks = writeChunksMode WriteMode
259-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
262-- Note that if you use a chunk size less than 8K (GHC's default buffer
263-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
264-- do not want buffering to occur at GHC level as well. Same thing applies to
267-- | Like 'write' but provides control over the write buffer. Output will
268-- be written to the IO device as soon as we collect the specified number of
272{-# INLINE writeWithBufferOf #-}
273writeWithBufferOf :: (MonadAsync m, MonadCatch m)
274 => Int -> FilePath -> SerialT m Word8 -> m ()
275writeWithBufferOf n file xs = writeChunks file $ AS.arraysOf n xs
277-- > write = 'writeWithBufferOf' defaultChunkSize
279-- | Write a byte stream to a file. Combines the bytes in chunks of size
280-- up to 'A.defaultChunkSize' before writing. If the file exists it is
281-- truncated to zero size before writing. If the file does not exist it is
282-- created. File is locked using single writer locking mode.
286write :: (MonadAsync m, MonadCatch m) => FilePath -> SerialT m Word8 -> m ()
287write = writeWithBufferOf defaultChunkSize
291write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
292write = toHandleWith A.defaultChunkSize
295-- | Append a stream of arrays to a file.
298{-# INLINE appendChunks #-}
299appendChunks :: (MonadAsync m, MonadCatch m, Storable a)
300 => FilePath -> SerialT m (Array a) -> m ()
301appendChunks = writeChunksMode AppendMode
303-- | Like 'append' but provides control over the write buffer. Output will
304-- be written to the IO device as soon as we collect the specified number of
308{-# INLINE appendWithBufferOf #-}
309appendWithBufferOf :: (MonadAsync m, MonadCatch m)
310 => Int -> FilePath -> SerialT m Word8 -> m ()
311appendWithBufferOf n file xs = appendChunks file $ AS.arraysOf n xs
313-- | Append a byte stream to a file. Combines the bytes in chunks of size up to
314-- 'A.defaultChunkSize' before writing. If the file exists then the new data
315-- is appended to the file. If the file does not exist it is created. File is
316-- locked using single writer locking mode.
320append :: (MonadAsync m, MonadCatch m) => FilePath -> SerialT m Word8 -> m ()
321append = appendWithBufferOf defaultChunkSize
324-- | Like 'append' but the file is not locked for exclusive writes.
327{-# INLINE appendShared #-}
328appendShared :: MonadIO m => Handle -> SerialT m Word8 -> m ()
329appendShared = undefined
332-------------------------------------------------------------------------------
333-- IO with encoding/decoding Unicode characters
334-------------------------------------------------------------------------------
338-- > readUtf8 = decodeUtf8 . read
340-- Read a UTF8 encoded stream of unicode characters from a file handle.
343{-# INLINE readUtf8 #-}
344readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
345readUtf8 = decodeUtf8 . read
348-- > writeUtf8 h s = write h $ encodeUtf8 s
350-- Encode a stream of unicode characters to UTF8 and write it to the given file
351-- handle. Default block buffering applies to the writes.
354{-# INLINE writeUtf8 #-}
355writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
356writeUtf8 h s = write h $ encodeUtf8 s
358-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
359-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
360-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
361-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
365{-# INLINE writeUtf8ByLines #-}
366writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
367writeUtf8ByLines = undefined
369-- | Read UTF-8 lines from a file handle and apply the specified fold to each
370-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
373{-# INLINE readLines #-}
374readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
375readLines h f = foldLines (readUtf8 h) f
377-------------------------------------------------------------------------------
378-- Framing on a sequence
379-------------------------------------------------------------------------------
381-- | Read a stream from a file handle and split it into frames delimited by
382-- the specified sequence of elements. The supplied fold is applied on each
386{-# INLINE readFrames #-}
387readFrames :: (IsStream t, MonadIO m, Storable a)
388 => Array a -> Handle -> Fold m a b -> t m b
389readFrames = undefined -- foldFrames . read
391-- | Write a stream to the given file handle buffering up to frames separated
392-- by the given sequence or up to a maximum of @defaultChunkSize@.
395{-# INLINE writeByFrames #-}
396writeByFrames :: (IsStream t, MonadIO m, Storable a)
397 => Array a -> Handle -> t m a -> m ()
398writeByFrames = undefined
400-------------------------------------------------------------------------------
401-- Random Access IO (Seek)
402-------------------------------------------------------------------------------
404-- XXX handles could be shared, so we may not want to use the handle state at
405-- all for these APIs. we can use pread and pwrite instead. On windows we will
406-- need to use readFile/writeFile with an offset argument.
408-------------------------------------------------------------------------------
410-- | Read the element at the given index treating the file as an array.
413{-# INLINE readIndex #-}
414readIndex :: Storable a => Handle -> Int -> Maybe a
415readIndex arr i = undefined
417-- NOTE: To represent a range to read we have chosen (start, size) instead of
418-- (start, end). This helps in removing the ambiguity of whether "end" is
419-- included in the range or not.
421-- We could avoid specifying the range to be read and instead use "take size"
422-- on the stream, but it may end up reading more and then consume it partially.
424-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
425-- from @handle@ starting at the offset @pos@ from the beginning of the file.
427-- Reads are performed in chunks of size @chunkSize@. For block devices, to
428-- avoid reading partial blocks @chunkSize@ must align with the block size of
429-- the underlying device. If the underlying block size is unknown, it is a good
430-- idea to keep it a multiple 4KiB. This API ensures that the start of each
431-- chunk is aligned with @chunkSize@ from second chunk onwards.
433{-# INLINE readSliceWith #-}
434readSliceWith :: (IsStream t, MonadIO m, Storable a)
435 => Int -> Handle -> Int -> Int -> t m a
436readSliceWith chunkSize h pos len = undefined
438-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
439-- at index @i@ and reading up to @count@ elements in the forward direction
440-- ending at the index @i + count - 1@.
443{-# INLINE readSlice #-}
444readSlice :: (IsStream t, MonadIO m, Storable a)
445 => Handle -> Int -> Int -> t m a
446readSlice = readSliceWith defaultChunkSize
448-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
449-- at index @i@ and reading up to @count@ elements in the reverse direction
450-- ending at the index @i - count + 1@.
453{-# INLINE readSliceRev #-}
454readSliceRev :: (IsStream t, MonadIO m, Storable a)
455 => Handle -> Int -> Int -> t m a
456readSliceRev h i count = undefined
458-- | Write the given element at the given index in the file.
461{-# INLINE writeIndex #-}
462writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
463writeIndex h i a = undefined
465-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
466-- starting at index @i@ and writing up to @count@ elements in the forward
467-- direction ending at the index @i + count - 1@.
470{-# INLINE writeSlice #-}
471writeSlice :: (IsStream t, Monad m, Storable a)
472 => Handle -> Int -> Int -> t m a -> m ()
473writeSlice h i len s = undefined
475-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
476-- starting at index @i@ and writing up to @count@ elements in the reverse
477-- direction ending at the index @i - count + 1@.
480{-# INLINE writeSliceRev #-}
481writeSliceRev :: (IsStream t, Monad m, Storable a)
482 => Handle -> Int -> Int -> t m a -> m ()
483writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list