4-- Module : Streamly.Internal.FileSystem.FD
5-- Copyright : (c) 2019 Composewell Technologies
8-- Maintainer : streamly@composewell.com
9-- Stability : experimental
12-- This module is a an experimental replacement for
13-- "Streamly.FileSystem.Handle". The former module provides IO facilities based
14-- on the GHC Handle type. The APIs in this module avoid the GHC handle layer
15-- and provide more explicit control over buffering.
17-- Read and write data as streams and arrays to and from files.
19-- This module provides read and write APIs based on handles. Before reading or
20-- writing, a file must be opened first using 'openFile'. The 'Handle' returned
21-- by 'openFile' is then used to access the file. A 'Handle' is backed by an
22-- operating system file descriptor. When the 'Handle' is garbage collected the
23-- underlying file descriptor is automatically closed. A handle can be
24-- explicitly closed using 'closeFile'.
26-- Reading and writing APIs are divided into two categories, sequential
27-- streaming APIs and random or seekable access APIs. File IO APIs are quite
28-- similar to "Streamly.Data.Array.Foreign" read write APIs. In that regard, arrays can
29-- be considered as in-memory files or files can be considered as on-disk
32-- > import qualified Streamly.Internal.FileSystem.FD as FD
35module Streamly.Internal.FileSystem.FD
44 -- TODO file path based APIs
49 -- | Streaming APIs read or write data to or from a file or device
50 -- sequentially, they never perform a seek to a random location. When
51 -- reading, the stream is lazy and generated on-demand as the consumer
52 -- consumes it. Read IO requests to the IO device are performed in chunks
53 -- of 32KiB, this is referred to as @defaultChunkSize@ in the
54 -- documentation. One IO request may or may not read the full chunk. If the
55 -- whole stream is not consumed, it is possible that we may read slightly
56 -- more from the IO device than what the consumer needed. Unless specified
57 -- otherwise in the API, writes are collected into chunks of
58 -- @defaultChunkSize@ before they are written to the IO device.
60 -- Streaming APIs work for all kind of devices, seekable or non-seekable;
61 -- including disks, files, memory devices, terminals, pipes, sockets and
62 -- fifos. While random access APIs work only for files or devices that have
63 -- random access or seek capability for example disks, memory devices.
64 -- Devices like terminals, pipes, sockets and fifos do not have random
67 -- ** Read File to Stream
82 -- ** Write File from Stream
92 , writeArraysPackedUpto
94 -- XXX these are incomplete
96 -- , writevArraysPackedUpto
98 -- -- * Random Access (Seek)
99 -- -- | Unlike the streaming APIs listed above, these APIs apply to devices or
100 -- files that have random access or seek capability. This type of devices
101 -- include disks, files, memory devices and exclude terminals, pipes,
102 -- sockets and fifos.
107 -- , readAt -- read from a given position to th end of file
108 -- , readSliceArrayUpto
109 -- , readSliceArrayOf
114 -- , writeAt -- start writing at the given position
119import Control.Monad.IO.Class (MonadIO(..))
120import Data.Word (Word8)
121-- import Foreign.ForeignPtr.Unsafe (unsafeForeignPtrToPtr)
122import Foreign.Ptr (castPtr)
123import Streamly.Internal.Data.Unboxed (Unbox)
124-- import System.IO (Handle, hGetBufSome, hPutBuf)
125import System.IO (IOMode)
126import Prelude hiding (read)
128import qualified GHC.IO.FD as FD
129import qualified GHC.IO.Device as RawIO
131import Streamly.Internal.Data.Array.Type
132 (Array(..), byteLength, unsafeFreeze, asPtrUnsafe)
133import Streamly.Internal.System.IO (defaultChunkSize)
134import Streamly.Internal.Data.Stream.Type (Stream)
135#if !defined(mingw32_HOST_OS)
137import Streamly.Internal.Data.Stream.IsStream.Type (toStreamD)
138import Streamly.Internal.System.IOVec (groupIOVecsOf)
139import qualified Streamly.Internal.FileSystem.FDIO as RawIO hiding (write)
140import qualified Streamly.Internal.System.IOVec.Type as RawIO
143-- import Streamly.Data.Fold (Fold)
144-- import Streamly.String (encodeUtf8, decodeUtf8, foldLines)
146import qualified Streamly.Data.Array as A
147import qualified Streamly.Data.Fold as FL
148import qualified Streamly.Internal.Data.Array.Mut as MArray
149 (Array(..), newPinnedBytes, asPtrUnsafe)
150import qualified Streamly.Internal.Data.Stream.Chunked as AS
151import qualified Streamly.Internal.Data.Stream as S
152import qualified Streamly.Internal.Data.Stream.StreamD.Type as D
153 (Stream(..), Step(..))
154import qualified Streamly.Internal.Data.Stream.StreamK.Type as K (mkStream)
157-------------------------------------------------------------------------------
159-------------------------------------------------------------------------------
161-- The following references may be useful to build an understanding about the
164-- http://www.linux-mag.com/id/308/ for blocking/non-blocking IO on linux.
165-- https://lwn.net/Articles/612483/ Non-blocking buffered file read operations
166-- https://en.wikipedia.org/wiki/C_file_input/output for C APIs.
167-- https://docs.oracle.com/javase/tutorial/essential/io/file.html for Java API.
168-- https://www.w3.org/TR/FileAPI/ for http file API.
170-------------------------------------------------------------------------------
172-------------------------------------------------------------------------------
174-- XXX attach a finalizer
175-- | A 'Handle' is returned by 'openFile' and is subsequently used to perform
176-- read and write operations on a file.
178newtype Handle = Handle FD.FD
180-- | File handle for standard input
182stdin = Handle FD.stdin
184-- | File handle for standard output
186stdout = Handle FD.stdout
188-- | File handle for standard error
190stderr = Handle FD.stderr
192-- XXX we can support all the flags that the "open" system call supports.
193-- Instead of using RTS locking mechanism can we use system provided locking
196-- | Open a file that is not a directory and return a file handle.
197-- 'openFile' enforces a multiple-reader single-writer locking on files. That
198-- is, there may either be many handles on the same file which manage input, or
199-- just one handle on the file which manages output. If any open handle is
200-- managing a file for output, no new handle can be allocated for that file. If
201-- any open handle is managing a file for input, new handles can only be
202-- allocated if they do not manage output. Whether two files are the same is
203-- implementation-dependent, but they should normally be the same if they have
204-- the same absolute path name and neither has been renamed, for example.
206openFile :: FilePath -> IOMode -> IO Handle
207openFile path mode = Handle . fst <$> FD.openFile path mode True
209-------------------------------------------------------------------------------
211-------------------------------------------------------------------------------
213-- | Read a 'ByteArray' from a file handle. If no data is available on the
214-- handle it blocks until some data becomes available. If data is available
215-- then it immediately returns that data without blocking. It reads a maximum
216-- of up to the size requested.
217{-# INLINABLE readArrayUpto #-}
218readArrayUpto :: Int -> Handle -> IO (Array Word8)
219readArrayUpto size (Handle fd) = do
220 arr <- MArray.newPinnedBytes size
221 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
222 MArray.asPtrUnsafe arr $ \p -> do
223 -- n <- hGetBufSome h p size
224#if MIN_VERSION_base(4,15,0)
225 n <- RawIO.read fd p 0 size
227 n <- RawIO.read fd p size
229 -- XXX shrink only if the diff is significant
230 -- Use unsafeFreezeWithShrink
233 $ arr { MArray.arrEnd = n, MArray.arrBound = size }
235-------------------------------------------------------------------------------
237-------------------------------------------------------------------------------
239-- | Write an 'Array' to a file handle.
242{-# INLINABLE writeArray #-}
243writeArray :: Unbox a => Handle -> Array a -> IO ()
244writeArray _ arr | A.length arr == 0 = return ()
245writeArray (Handle fd) arr =
246 asPtrUnsafe arr $ \p ->
247 -- RawIO.writeAll fd (castPtr p) aLen
248#if MIN_VERSION_base(4,15,0)
249 RawIO.write fd (castPtr p) 0 aLen
251 RawIO.write fd (castPtr p) aLen
254 -- Experiment to compare "writev" based IO with "write" based IO.
256 let iov' = iov {arrEnd = arrBound iov}
257 A.writeIndex iov' 0 (RawIO.IOVec (castPtr p) (fromIntegral aLen))
258 RawIO.writevAll fd (unsafeForeignPtrToPtr (aStart iov')) 1
261 aLen = byteLength arr
263#if !defined(mingw32_HOST_OS)
265-- | Write an array of 'IOVec' to a file handle.
268{-# INLINABLE writeIOVec #-}
269writeIOVec :: Handle -> Array RawIO.IOVec -> IO ()
270writeIOVec _ iov | A.length iov == 0 = return ()
271writeIOVec (Handle fd) iov =
272 asPtrUnsafe iov $ \p ->
273 RawIO.writevAll fd p (A.length iov)
277-------------------------------------------------------------------------------
278-- Stream of Arrays IO
279-------------------------------------------------------------------------------
281-- | @readArraysOfUpto size h@ reads a stream of arrays from file handle @h@.
282-- The maximum size of a single array is specified by @size@. The actual size
283-- read may be less than or equal to @size@.
284{-# INLINE _readArraysOfUpto #-}
285_readArraysOfUpto :: (MonadIO m)
286 => Int -> Handle -> Stream m (Array Word8)
287_readArraysOfUpto size h = S.fromStreamK go
289 -- XXX use cons/nil instead
290 go = K.mkStream $ \_ yld _ stp -> do
291 arr <- liftIO $ readArrayUpto size h
296{-# INLINE_NORMAL readArraysOfUpto #-}
297readArraysOfUpto :: (MonadIO m)
298 => Int -> Handle -> Stream m (Array Word8)
299readArraysOfUpto size h = S.fromStreamD (D.Stream step ())
301 {-# INLINE_LATE step #-}
303 arr <- liftIO $ readArrayUpto size h
309-- XXX read 'Array a' instead of Word8
311-- | @readArrays h@ reads a stream of arrays from file handle @h@.
312-- The maximum size of a single array is limited to @defaultChunkSize@.
313-- 'readArrays' ignores the prevailing 'TextEncoding' and 'NewlineMode'
316-- > readArrays = readArraysOfUpto defaultChunkSize
319{-# INLINE readArrays #-}
320readArrays :: (MonadIO m) => Handle -> Stream m (Array Word8)
321readArrays = readArraysOfUpto defaultChunkSize
323-------------------------------------------------------------------------------
324-- Read File to Stream
325-------------------------------------------------------------------------------
327-- TODO for concurrent streams implement readahead IO. We can send multiple
328-- read requests at the same time. For serial case we can use async IO. We can
329-- also control the read throughput in mbps or IOPS.
331-- | @readInChunksOf chunkSize handle@ reads a byte stream from a file handle,
332-- reads are performed in chunks of up to @chunkSize@. The stream ends as soon
333-- as EOF is encountered.
335{-# INLINE readInChunksOf #-}
336readInChunksOf :: (MonadIO m) => Int -> Handle -> Stream m Word8
337readInChunksOf chunkSize h = AS.concat $ readArraysOfUpto chunkSize h
340-- read :: (MonadIO m, Unbox a) => Handle -> Stream m a
342-- > read = 'readByChunks' A.defaultChunkSize
343-- | Generate a stream of elements of the given type from a file 'Handle'. The
344-- stream ends when EOF is encountered.
348read :: (MonadIO m) => Handle -> Stream m Word8
349read = AS.concat . readArrays
351-------------------------------------------------------------------------------
353-------------------------------------------------------------------------------
355-- | Write a stream of arrays to a handle.
358{-# INLINE writeArrays #-}
359writeArrays :: (MonadIO m, Unbox a) => Handle -> Stream m (Array a) -> m ()
360writeArrays h = S.fold (FL.drainBy (liftIO . writeArray h))
362-- | Write a stream of arrays to a handle after coalescing them in chunks of
363-- specified size. The chunk size is only a maximum and the actual writes could
364-- be smaller than that as we do not split the arrays to fit them to the
368{-# INLINE writeArraysPackedUpto #-}
369writeArraysPackedUpto :: (MonadIO m, Unbox a)
370 => Int -> Handle -> Stream m (Array a) -> m ()
371writeArraysPackedUpto n h xs = writeArrays h $ AS.compact n xs
373#if !defined(mingw32_HOST_OS)
375-- XXX this is incomplete
376-- | Write a stream of 'IOVec' arrays to a handle.
380writev :: MonadIO m => Handle -> Stream m (Array RawIO.IOVec) -> m ()
381writev h = S.mapM_ (liftIO . writeIOVec h)
383-- XXX this is incomplete
384-- | Write a stream of arrays to a handle after grouping them in 'IOVec' arrays
385-- of up to a maximum total size. Writes are performed using gather IO via
386-- @writev@ system call. The maximum number of entries in each 'IOVec' group
390{-# INLINE _writevArraysPackedUpto #-}
391_writevArraysPackedUpto :: MonadIO m
392 => Int -> Handle -> Stream m (Array a) -> m ()
393_writevArraysPackedUpto n h xs =
394 writev h $ fromStreamD $ groupIOVecsOf n 512 (toStreamD xs)
398-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
401-- Note that if you use a chunk size less than 8K (GHC's default buffer
402-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
403-- do not want buffering to occur at GHC level as well. Same thing applies to
406-- | Like 'write' but provides control over the write buffer. Output will
407-- be written to the IO device as soon as we collect the specified number of
411{-# INLINE writeInChunksOf #-}
412writeInChunksOf :: MonadIO m => Int -> Handle -> Stream m Word8 -> m ()
413writeInChunksOf n h m = writeArrays h $ AS.arraysOf n m
415-- > write = 'writeInChunksOf' A.defaultChunkSize
417-- | Write a byte stream to a file handle. Combines the bytes in chunks of size
418-- up to 'A.defaultChunkSize' before writing. Note that the write behavior
419-- depends on the 'IOMode' and the current seek position of the handle.
423write :: MonadIO m => Handle -> Stream m Word8 -> m ()
424write = writeInChunksOf defaultChunkSize
428write :: (MonadIO m, Unboxed a) => Handle -> Stream m a -> m ()
429write = toHandleWith A.defaultChunkSize
432-------------------------------------------------------------------------------
433-- IO with encoding/decoding Unicode characters
434-------------------------------------------------------------------------------
438-- > readUtf8 = decodeUtf8 . read
440-- Read a UTF8 encoded stream of unicode characters from a file handle.
443{-# INLINE readUtf8 #-}
444readUtf8 :: (MonadIO m) => Handle -> Stream m Char
445readUtf8 = decodeUtf8 . read
448-- > writeUtf8 h s = write h $ encodeUtf8 s
450-- Encode a stream of unicode characters to UTF8 and write it to the given file
451-- handle. Default block buffering applies to the writes.
454{-# INLINE writeUtf8 #-}
455writeUtf8 :: MonadIO m => Handle -> Stream m Char -> m ()
456writeUtf8 h s = write h $ encodeUtf8 s
458-- | Write a stream of unicode characters after encoding to UTF-8 in chunks
459-- separated by a linefeed character @'\n'@. If the size of the buffer exceeds
460-- @defaultChunkSize@ and a linefeed is not yet found, the buffer is written
461-- anyway. This is similar to writing to a 'Handle' with the 'LineBuffering'
465{-# INLINE writeUtf8ByLines #-}
466writeUtf8ByLines :: (MonadIO m) => Handle -> Stream m Char -> m ()
467writeUtf8ByLines = undefined
469-- | Read UTF-8 lines from a file handle and apply the specified fold to each
470-- line. This is similar to reading a 'Handle' with the 'LineBuffering' option.
473{-# INLINE readLines #-}
474readLines :: (MonadIO m) => Handle -> Fold m Char b -> Stream m b
475readLines h f = foldLines (readUtf8 h) f
477-------------------------------------------------------------------------------
478-- Framing on a sequence
479-------------------------------------------------------------------------------
481-- | Read a stream from a file handle and split it into frames delimited by
482-- the specified sequence of elements. The supplied fold is applied on each
486{-# INLINE readFrames #-}
487readFrames :: (MonadIO m, Unboxed a)
488 => Array a -> Handle -> Fold m a b -> Stream m b
489readFrames = undefined -- foldFrames . read
491-- | Write a stream to the given file handle buffering up to frames separated
492-- by the given sequence or up to a maximum of @defaultChunkSize@.
495{-# INLINE writeByFrames #-}
496writeByFrames :: (MonadIO m, Unboxed a)
497 => Array a -> Handle -> Stream m a -> m ()
498writeByFrames = undefined
500-------------------------------------------------------------------------------
501-- Random Access IO (Seek)
502-------------------------------------------------------------------------------
504-- XXX handles could be shared, so we may not want to use the handle state at
505-- all for these APIs. we can use pread and pwrite instead. On windows we will
506-- need to use readFile/writeFile with an offset argument.
508-------------------------------------------------------------------------------
510-- | Read the element at the given index treating the file as an array.
513{-# INLINE readIndex #-}
514readIndex :: Unboxed a => Handle -> Int -> Maybe a
515readIndex arr i = undefined
517-- NOTE: To represent a range to read we have chosen (start, size) instead of
518-- (start, end). This helps in removing the ambiguity of whether "end" is
519-- included in the range or not.
521-- We could avoid specifying the range to be read and instead use "take size"
522-- on the stream, but it may end up reading more and then consume it partially.
524-- | @readSliceWith chunkSize handle pos len@ reads up to @len@ bytes
525-- from @handle@ starting at the offset @pos@ from the beginning of the file.
527-- Reads are performed in chunks of size @chunkSize@. For block devices, to
528-- avoid reading partial blocks @chunkSize@ must align with the block size of
529-- the underlying device. If the underlying block size is unknown, it is a good
530-- idea to keep it a multiple 4KiB. This API ensures that the start of each
531-- chunk is aligned with @chunkSize@ from second chunk onwards.
533{-# INLINE readSliceWith #-}
534readSliceWith :: (MonadIO m, Unboxed a)
535 => Int -> Handle -> Int -> Int -> Stream m a
536readSliceWith chunkSize h pos len = undefined
538-- | @readSlice h i count@ streams a slice from the file handle @h@ starting
539-- at index @i@ and reading up to @count@ elements in the forward direction
540-- ending at the index @i + count - 1@.
543{-# INLINE readSlice #-}
544readSlice :: (MonadIO m, Unboxed a)
545 => Handle -> Int -> Int -> Stream m a
546readSlice = readSliceWith defaultChunkSize
548-- | @readSliceRev h i count@ streams a slice from the file handle @h@ starting
549-- at index @i@ and reading up to @count@ elements in the reverse direction
550-- ending at the index @i - count + 1@.
553{-# INLINE readSliceRev #-}
554readSliceRev :: (MonadIO m, Unboxed a)
555 => Handle -> Int -> Int -> Stream m a
556readSliceRev h i count = undefined
558-- | Write the given element at the given index in the file.
561{-# INLINE writeIndex #-}
562writeIndex :: (MonadIO m, Unboxed a) => Handle -> Int -> a -> m ()
563writeIndex h i a = undefined
565-- | @writeSlice h i count stream@ writes a stream to the file handle @h@
566-- starting at index @i@ and writing up to @count@ elements in the forward
567-- direction ending at the index @i + count - 1@.
570{-# INLINE writeSlice #-}
571writeSlice :: (Monad m, Unboxed a)
572 => Handle -> Int -> Int -> Stream m a -> m ()
573writeSlice h i len s = undefined
575-- | @writeSliceRev h i count stream@ writes a stream to the file handle @h@
576-- starting at index @i@ and writing up to @count@ elements in the reverse
577-- direction ending at the index @i - count + 1@.
580{-# INLINE writeSliceRev #-}
581writeSliceRev :: (Monad m, Unboxed a)
582 => Handle -> Int -> Int -> Stream m a -> m ()
583writeSliceRev arr i len s = undefined
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list