2{-# LANGUAGE BangPatterns #-}
3{-# LANGUAGE MagicHash #-}
4{-# LANGUAGE RecordWildCards #-}
5{-# LANGUAGE UnboxedTuples #-}
10-- Module : Streamly.FileSystem.File
11-- Copyright : (c) 2018 Harendra Kumar
14-- Maintainer : harendra.kumar@gmail.com
15-- Stability : experimental
18-- Read and write streams and arrays to and from files. File IO APIs are quite
19-- similar to "Streamly.Mem.Array" read write APIs. In that regard, arrays can
20-- be considered as in-memory files or files can be considered as on-disk
21-- arrays. IO APIs are divided into two categories, sequential streaming IO
22-- APIs and random access IO APIs. Control over the file reading and writing
23-- behavior in terms of buffering, encoding, decoding is in the hands of the
24-- programmer, the 'TextEncoding', 'NewLineMode', and 'Buffering' options of
25-- the underlying handle provided by GHC are not needed and ignored.
27-- > import qualified Streamly.FileSystem.File as File
30module Streamly.FileSystem.File
33 -- | Stream data to or from a file or device sequentially. When reading,
34 -- the stream is lazy and generated on-demand as the consumer consumes it.
35 -- Read IO requests to the IO device are performed in chunks limited to a
36 -- maximum size of 32KiB, this is referred to as @defaultChunkSize@ in the
37 -- documentation. One IO request may or may not read the full
38 -- chunk. If the whole stream is not consumed, it is possible that we may
39 -- read slightly more from the IO device than what the consumer needed.
40 -- Unless specified otherwise in the API, writes are collected into chunks
41 -- of @defaultChunkSize@ before they are written to the IO device.
43 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
44 -- including disks, files, memory devices, terminals, pipes, sockets and
45 -- fifos. While random access APIs work only for files or devices that have
46 -- random access or seek capability for example disks, memory devices.
47 -- Devices like terminals, pipes, sockets and fifos do not have random
50 -- TODO file path based APIs
54 -- ** Read File to Stream
69 -- ** Write File from Stream
80 -- -- * Random Access (Seek)
81 -- -- | Unlike the streaming APIs listed above, these APIs apply to devices or
82 -- files that have random access or seek capability. This type of devices
83 -- include disks, files, memory devices and exclude terminals, pipes,
89 -- , readAt -- read from a given position to th end of file
90 -- , readSliceArrayUpto
96 -- , writeAt -- start writing at the given position
101import Control.Monad.IO.Class (MonadIO(..))
102import Data.Word (Word8)
103import Foreign.ForeignPtr (withForeignPtr)
104import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
105import Foreign.Ptr (minusPtr, plusPtr)
106import Foreign.Storable (Storable(..))
107import GHC.ForeignPtr (mallocPlainForeignPtrBytes)
108import System.IO (Handle, hGetBufSome, hPutBuf)
109import Prelude hiding (read)
111import Streamly.Mem.Array.Types (Array(..))
112import Streamly.Streams.Serial (SerialT)
113import Streamly.Streams.StreamK.Type (IsStream, mkStream)
114-- import Streamly.Fold (Fold)
115-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
117import qualified Streamly.Mem.Array as A
118import qualified Streamly.Mem.Array.Types as A hiding (flattenArrays)
119import qualified Streamly.Prelude as S
121-------------------------------------------------------------------------------
123-------------------------------------------------------------------------------
125-- The following references may be useful to build an understanding about the
128-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
129-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
130-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
131-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
132-- https://www.w3.org/TR/FileAPI/ for http file API.
134-------------------------------------------------------------------------------
136-------------------------------------------------------------------------------
138-- | Read a 'ByteArray' from a file handle. If no data is available on the
139-- handle it blocks until some data becomes available. If data is available
140-- then it immediately returns that data without blocking. It reads a maximum
141-- of up to the size requested.
142{-# INLINABLE readArrayUpto #-}
143readArrayUpto :: Int -> Handle -> IO (Array Word8)
144readArrayUpto size h = do
145 ptr <- mallocPlainForeignPtrBytes size
146 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
147 withForeignPtr ptr $ \p -> do
148 n <- hGetBufSome h p size
151 , aEnd = p `plusPtr` n
152 , aBound = p `plusPtr` size
154 -- XXX shrink only if the diff is significant
157-------------------------------------------------------------------------------
159-------------------------------------------------------------------------------
161-- | Write an Array to a file handle.
164{-# INLINABLE writeArray #-}
165writeArray :: Storable a => Handle -> Array a -> IO ()
166writeArray _ arr | A.length arr == 0 = return ()
167writeArray h Array{..} = withForeignPtr aStart $ \p -> hPutBuf h p aLen
170 let p = unsafeForeignPtrToPtr aStart
173-------------------------------------------------------------------------------
174-- Stream of Arrays IO
175-------------------------------------------------------------------------------
177-- | @readArraysUpto size h@ reads a stream of arrays from file handle @h@.
178-- The maximum size of a single array is limited to @size@.
179-- 'fromHandleArraysUpto' ignores the prevailing 'TextEncoding' and 'NewlineMode'
181{-# INLINABLE readArraysUpto #-}
182readArraysUpto :: (IsStream t, MonadIO m)
183 => Int -> Handle -> t m (Array Word8)
184readArraysUpto size h = go
186 -- XXX use cons/nil instead
187 go = mkStream $ \_ yld sng _ -> do
188 arr <- liftIO $ readArrayUpto size h
189 if A.length arr < size
193-- XXX read 'Array a' instead of Word8
195-- | @readArrays h@ reads a stream of arrays from file handle @h@.
196-- The maximum size of a single array is limited to @defaultChunkSize@.
197-- 'readArrays' ignores the prevailing 'TextEncoding' and 'NewlineMode'
201{-# INLINE readArrays #-}
202readArrays :: (IsStream t, MonadIO m) => Handle -> t m (Array Word8)
203readArrays = readArraysUpto A.defaultChunkSize
205-------------------------------------------------------------------------------
206-- Read File to Stream
207-------------------------------------------------------------------------------
209-- TODO for concurrent streams implement readahead IO. We can send multiple
210-- read requests at the same time. For serial case we can use async IO. We can
211-- also control the read throughput in mbps or IOPS.
214-- | @readByChunksUpto chunkSize handle@ reads a byte stream from a file
215-- handle, reads are performed in chunks of up to @chunkSize@. The stream ends
216-- as soon as EOF is encountered.
218{-# INLINE readByChunksUpto #-}
219readByChunksUpto :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8
220readByChunksUpto chunkSize h = A.flattenArrays $ readArraysUpto chunkSize h
224-- read :: (IsStream t, MonadIO m, Storable a) => Handle -> t m a
226-- > read = 'readByChunks' A.defaultChunkSize
227-- | Generate a stream of elements of the given type from a file 'Handle'. The
228-- stream ends when EOF is encountered.
232read :: (IsStream t, MonadIO m) => Handle -> t m Word8
233read = A.flattenArrays . readArrays
235-------------------------------------------------------------------------------
237-------------------------------------------------------------------------------
239-- | Write a stream of arrays to a handle.
242{-# INLINE writeArrays #-}
243writeArrays :: (MonadIO m, Storable a) => Handle -> SerialT m (Array a) -> m ()
244writeArrays h m = S.mapM_ (liftIO . writeArray h) m
246-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
249-- Note that if you use a chunk size less than 8K (GHC's default buffer
250-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
251-- do not want buffering to occur at GHC level as well. Same thing applies to
254-- | Like 'write' but provides control over the write buffer. Output will
255-- be written to the IO device as soon as we collect the specified number of
259{-# INLINE writeByChunks #-}
260writeByChunks :: MonadIO m => Int -> Handle -> SerialT m Word8 -> m ()
261writeByChunks n h m = writeArrays h $ A.arraysOf n m
263-- > write = 'writeByChunks' A.defaultChunkSize
265-- | Write a byte stream to a file handle. Combines the bytes in chunks of size
266-- up to 'A.defaultChunkSize' before writing. Note that the write behavior
267-- depends on the 'IOMode' and the current seek position of the handle.
271write :: MonadIO m => Handle -> SerialT m Word8 -> m ()
272write = writeByChunks A.defaultChunkSize
276write :: (MonadIO m, Storable a) => Handle -> SerialT m a -> m ()
277write = toHandleWith A.defaultChunkSize
280-------------------------------------------------------------------------------
281-- IO with encoding/decoding Unicode characters
282-------------------------------------------------------------------------------
286-- > readUtf8 = decodeUtf8 . read
288-- Read a UTF8 encoded stream of unicode characters from a file handle.
291{-# INLINE readUtf8 #-}
292readUtf8 :: (IsStream t, MonadIO m) => Handle -> t m Char
293readUtf8 = decodeUtf8 . read
296-- > writeUtf8 h s = write h $ encodeUtf8 s
298-- Encode a stream of unicode characters to UTF8 and write it to the given file
299-- handle. Default block buffering applies to the writes.
302{-# INLINE writeUtf8 #-}
303writeUtf8 :: MonadIO m => Handle -> SerialT m Char -> m ()
304writeUtf8 h s = write h $ encodeUtf8 s
306-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
307-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
308-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
309-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
313{-# INLINE writeUtf8ByLines #-}
314writeUtf8ByLines :: (IsStream t, MonadIO m) => Handle -> t m Char -> m ()
315writeUtf8ByLines = undefined
317-- | Read UTF-8 lines from a file handle and apply the specified fold to each
318-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
321{-# INLINE readLines #-}
322readLines :: (IsStream t, MonadIO m) => Handle -> Fold m Char b -> t m b
323readLines h f = foldLines (readUtf8 h) f
325-------------------------------------------------------------------------------
326-- Framing on a sequence
327-------------------------------------------------------------------------------
329-- | Read a stream from a file handle and split it into frames delimited by
330-- the specified sequence of elements. The supplied fold is applied on each
334{-# INLINE readFrames #-}
335readFrames :: (IsStream t, MonadIO m, Storable a)
336 => Array a -> Handle -> Fold m a b -> t m b
337readFrames = undefined -- foldFrames . read
339-- | Write a stream to the given file handle buffering up to frames separated
340-- by the given sequence or up to a maximum of @defaultChunkSize@.
343{-# INLINE writeByFrames #-}
344writeByFrames :: (IsStream t, MonadIO m, Storable a)
345 => Array a -> Handle -> t m a -> m ()
346writeByFrames = undefined
348-------------------------------------------------------------------------------
349-- Random Access IO (Seek)
350-------------------------------------------------------------------------------
352-- XXX handles could be shared, so we may not want to use the handle state at
353-- all for these APIs. we can use pread and pwrite instead. On windows we will
354-- need to use readFile/writeFile with an offset argument.
356-------------------------------------------------------------------------------
358-- | Read the element at the given index treating the file as an array.
361{-# INLINE readIndex #-}
362readIndex :: Storable a => Handle -> Int -> Maybe a
363readIndex arr i = undefined
365-- NOTE: To represent a range to read we have chosen (start, size) instead of
366-- (start, end). This helps in removing the ambiguity of whether "end" is
367-- included in the range or not.
369-- We could avoid specifying the range to be read and instead use "take size"
370-- on the stream, but it may end up reading more and then consume it partially.
372-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
373-- from @handle@ starting at the offset @pos@ from the beginning of the file.
375-- Reads are performed in chunks of size @chunkSize@. For block devices, to
376-- avoid reading partial blocks @chunkSize@ must align with the block size of
377-- the underlying device. If the underlying block size is unknown, it is a good
378-- idea to keep it a multiple 4KiB. This API ensures that the start of each
379-- chunk is aligned with @chunkSize@ from second chunk onwards.
381{-# INLINE readSliceWith #-}
382readSliceWith :: (IsStream t, MonadIO m, Storable a)
383 => Int -> Handle -> Int -> Int -> t m a
384readSliceWith chunkSize h pos len = undefined
386-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
387-- at index @i@ and reading up to @count@ elements in the forward direction
388-- ending at the index @i + count - 1@.
391{-# INLINE readSlice #-}
392readSlice :: (IsStream t, MonadIO m, Storable a)
393 => Handle -> Int -> Int -> t m a
394readSlice = readSliceWith A.defaultChunkSize
396-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
397-- at index @i@ and reading up to @count@ elements in the reverse direction
398-- ending at the index @i - count + 1@.
401{-# INLINE readSliceRev #-}
402readSliceRev :: (IsStream t, MonadIO m, Storable a)
403 => Handle -> Int -> Int -> t m a
404readSliceRev h i count = undefined
406-- | Write the given element at the given index in the file.
409{-# INLINE writeIndex #-}
410writeIndex :: (MonadIO m, Storable a) => Handle -> Int -> a -> m ()
411writeIndex h i a = undefined
413-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
414-- starting at index @i@ and writing up to @count@ elements in the forward
415-- direction ending at the index @i + count - 1@.
418{-# INLINE writeSlice #-}
419writeSlice :: (IsStream t, Monad m, Storable a)
420 => Handle -> Int -> Int -> t m a -> m ()
421writeSlice h i len s = undefined
423-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
424-- starting at index @i@ and writing up to @count@ elements in the reverse
425-- direction ending at the index @i - count + 1@.
428{-# INLINE writeSliceRev #-}
429writeSliceRev :: (IsStream t, Monad m, Storable a)
430 => Handle -> Int -> Int -> t m a -> m ()
431writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list