4-- Module : Streamly.Internal.FileSystem.Dir
5-- Copyright : (c) 2018 Composewell Technologies
8-- Maintainer : streamly@composewell.com
9-- Stability : pre-release
12module Streamly.Internal.FileSystem.Dir
17 -- read not just the names but also the inode attrs of the children. This
18 -- abstraction makes sense because when we read the dir contents we also
19 -- get the inodes, and it is cheaper to get the attrs from the inodes
20 -- instead of resolving the paths and get those. This abstraction may be
21 -- less portable as different platforms may have different attrs. To
22 -- optimize, we can also add a filter/pattern/parser on the names of the
23 -- children that we want to read. We can call that readAttrsWith? Or just
24 -- have the default readAttrs do that? Usually we won't need that, so it
25 -- may be better to keep that a separate API.
28 -- recursive read requires us to read the attributes of the children to
29 -- determine if something is a dirctory or not. Therefore, it may be a good
30 -- idea to have a low level routine that also spits out the attributes of
31 -- the files, we get that for free. We can also add a filter/pattern/parser
32 -- on the names of the children that we want to read.
33 --, readAttrsRecursive -- Options: acyclic, follow symlinks
39 -- We can implement this in terms of readAttrsRecursive without losing
41 -- , readEitherRecursive -- Options: acyclic, follow symlinks
42 -- , readAncestors -- read the parent chain using the .. entry.
43 -- , readAncestorsAttrs
46 -- | Use the more convenient stream APIs instead of unfolds where possible.
54 , toStreamWithBufferOf
57 , readChunksWithBufferOf
59 , toChunksWithBufferOf
65 -- Byte stream write (Streams)
67 , fromStreamWithBufferOf
72 , writeChunksWithBufferOf
74 -- -- * Array stream Write
76 , fromChunksWithBufferOf
86import Control.Monad.IO.Class (MonadIO(..))
87import Data.Bifunctor (bimap)
88import Data.Either (isRight, isLeft, fromLeft, fromRight)
89import Streamly.Data.Stream (Stream)
90import Streamly.Internal.Data.Unfold.Type (Unfold(..))
91import System.FilePath ((</>))
93import qualified Streamly.Data.Unfold as UF
94import qualified Streamly.Internal.Data.Unfold as UF (mapM2)
95import qualified Streamly.Data.Stream as S
96import qualified System.Directory as Dir
98import Prelude hiding (read)
101{-# INLINABLE readArrayUpto #-}
102readArrayUpto :: Int -> Handle -> IO (Array Word8)
103readArrayUpto size h = do
104 ptr <- mallocPlainForeignPtrBytes size
105 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))
106 withForeignPtr ptr $ \p -> do
107 n <- hGetBufSome h p size
110 , arrEnd = p `plusPtr` n
111 , arrBound = p `plusPtr` size
113 -- XXX shrink only if the diff is significant
116-------------------------------------------------------------------------------
117-- Stream of Arrays IO
118-------------------------------------------------------------------------------
120-- | @toChunksWithBufferOf size h@ reads a stream of arrays from file handle @h@.
121-- The maximum size of a single array is specified by @size@. The actual size
122-- read may be less than or equal to @size@.
123{-# INLINE _toChunksWithBufferOf #-}
124_toChunksWithBufferOf :: MonadIO m => Int -> Handle -> Stream m (Array Word8)
125_toChunksWithBufferOf size h = go
127 -- XXX use cons/nil instead
128 go = mkStream $ \_ yld _ stp -> do
129 arr <- liftIO $ readArrayUpto size h
134-- | @toChunksWithBufferOf size handle@ reads a stream of arrays from the file
135-- handle @handle@. The maximum size of a single array is limited to @size@.
136-- The actual size read may be less than or equal to @size@.
139{-# INLINE_NORMAL toChunksWithBufferOf #-}
140toChunksWithBufferOf :: MonadIO m => Int -> Handle -> Stream m (Array Word8)
141toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ())
143 {-# INLINE_LATE step #-}
145 arr <- liftIO $ readArrayUpto size h
151-- | Unfold the tuple @(bufsize, handle)@ into a stream of 'Word8' arrays.
152-- Read requests to the IO device are performed using a buffer of size
153-- @bufsize@. The size of an array in the resulting stream is always less than
154-- or equal to @bufsize@.
157{-# INLINE_NORMAL readChunksWithBufferOf #-}
158readChunksWithBufferOf :: MonadIO m => Unfold m (Int, Handle) (Array Word8)
159readChunksWithBufferOf = Unfold step return
161 {-# INLINE_LATE step #-}
163 arr <- liftIO $ readArrayUpto size h
167 _ -> D.Yield arr (size, h)
169-- XXX read 'Array a' instead of Word8
171-- | @toChunks handle@ reads a stream of arrays from the specified file
172-- handle. The maximum size of a single array is limited to
173-- @defaultChunkSize@. The actual size read may be less than or equal to
174-- @defaultChunkSize@.
176-- > toChunks = toChunksWithBufferOf defaultChunkSize
179{-# INLINE toChunks #-}
180toChunks :: MonadIO m => Handle -> Stream m (Array Word8)
181toChunks = toChunksWithBufferOf defaultChunkSize
183-- | Unfolds a handle into a stream of 'Word8' arrays. Requests to the IO
184-- device are performed using a buffer of size
185-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize'. The
186-- size of arrays in the resulting stream are therefore less than or equal to
187-- 'Streamly.Internal.Data.Array.Type.defaultChunkSize'.
190{-# INLINE readChunks #-}
191readChunks :: MonadIO m => Unfold m Handle (Array Word8)
192readChunks = UF.first readChunksWithBufferOf defaultChunkSize
194-------------------------------------------------------------------------------
195-- Read a Directory to Stream
196-------------------------------------------------------------------------------
198-- TODO for concurrent streams implement readahead IO. We can send multiple
199-- read requests at the same time. For serial case we can use async IO. We can
200-- also control the read throughput in mbps or IOPS.
202-- | Unfolds the tuple @(bufsize, handle)@ into a byte stream, read requests
203-- to the IO device are performed using buffers of @bufsize@.
206{-# INLINE readWithBufferOf #-}
207readWithBufferOf :: MonadIO m => Unfold m (Int, Handle) Word8
208readWithBufferOf = UF.many readChunksWithBufferOf A.read
210-- | @toStreamWithBufferOf bufsize handle@ reads a byte stream from a file
211-- handle, reads are performed in chunks of up to @bufsize@.
214{-# INLINE toStreamWithBufferOf #-}
215toStreamWithBufferOf :: MonadIO m => Int -> Handle -> Stream m Word8
216toStreamWithBufferOf chunkSize h = AS.concat $ toChunksWithBufferOf chunkSize h
219-- read child node names from a dir filtering out . and ..
221-- . and .. are an implementation artifact, and should probably not be used in
222-- user level abstractions.
224-- . does not seem to have any useful purpose. If we have the path of the dir
225-- then we will resolve it to get the inode of the dir so the . entry would be
226-- redundant. If we have the inode of the dir to read the dir then it is
227-- redundant. Is this for cross check when doing fsck?
229-- For .. we have the readAncestors API, we should not have this in the
232-- XXX exception handling
234-- | Read a directory emitting a stream with names of the children. Filter out
235-- "." and ".." entries.
240reader :: MonadIO m => Unfold m FilePath FilePath
242 -- XXX use proper streaming read of the dir
243 UF.filter (\x -> x /= "." && x /= "..")
244 $ UF.lmapM (liftIO . Dir.getDirectoryContents) UF.fromList
246-- XXX We can use a more general mechanism to filter the contents of a
247-- directory. We can just stat each child and pass on the stat information. We
248-- can then use that info to do a general filtering. "find" like filters can be
251-- | Read directories as Left and files as Right. Filter out "." and ".."
256{-# INLINE eitherReader #-}
257eitherReader :: MonadIO m => Unfold m FilePath (Either FilePath FilePath)
258eitherReader = UF.mapM2 classify reader
263 r <- liftIO $ Dir.doesDirectoryExist (dir ++ "/" ++ x)
264 return $ if r then Left x else Right x
266{-# INLINE eitherReaderPaths #-}
267eitherReaderPaths :: MonadIO m => Unfold m FilePath (Either FilePath FilePath)
269 UF.mapM2 (\dir -> return . bimap (dir </>) (dir </>)) eitherReader
276{-# INLINE fileReader #-}
277fileReader :: MonadIO m => Unfold m FilePath FilePath
278fileReader = fmap (fromRight undefined) $ UF.filter isRight eitherReader
280-- | Read directories only. Filter out "." and ".." entries.
284{-# INLINE dirReader #-}
285dirReader :: MonadIO m => Unfold m FilePath FilePath
286dirReader = fmap (fromLeft undefined) $ UF.filter isLeft eitherReader
288-- | Raw read of a directory.
292read :: MonadIO m => FilePath -> Stream m FilePath
293read = S.unfold reader
295{-# DEPRECATED toStream "Please use 'read' instead" #-}
296{-# INLINE toStream #-}
297toStream :: MonadIO m => String -> Stream m String
300-- | Read directories as Left and files as Right. Filter out "." and ".."
301-- entries. The output contains the names of the directories and files.
304{-# INLINE readEither #-}
305readEither :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)
306readEither = S.unfold eitherReader
308-- | Like 'readEither' but prefix the names of the files and directories with
309-- the supplied directory path.
310{-# INLINE readEitherPaths #-}
311readEitherPaths :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)
312readEitherPaths dir = fmap (bimap (dir </>) (dir </>)) $ readEither dir
314{-# DEPRECATED toEither "Please use 'readEither' instead" #-}
315{-# INLINE toEither #-}
316toEither :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)
323{-# INLINE readFiles #-}
324readFiles :: MonadIO m => FilePath -> Stream m FilePath
325readFiles = S.unfold fileReader
327{-# DEPRECATED toFiles "Please use 'readFiles' instead" #-}
328{-# INLINE toFiles #-}
329toFiles :: MonadIO m => FilePath -> Stream m FilePath
332-- | Read directories only.
336{-# INLINE readDirs #-}
337readDirs :: MonadIO m => FilePath -> Stream m FilePath
338readDirs = S.unfold dirReader
340{-# DEPRECATED toDirs "Please use 'readDirs' instead" #-}
342toDirs :: MonadIO m => String -> Stream m String
346-------------------------------------------------------------------------------
348-------------------------------------------------------------------------------
350-------------------------------------------------------------------------------
352-------------------------------------------------------------------------------
354-- | Write an 'Array' to a file handle.
357{-# INLINABLE writeArray #-}
358writeArray :: Storable a => Handle -> Array a -> IO ()
359writeArray _ arr | A.length arr == 0 = return ()
360writeArray h Array{..} = withForeignPtr aStart $ \p -> hPutBuf h p aLen
363 let p = unsafeForeignPtrToPtr aStart
364 in arrEnd `minusPtr` p
366-------------------------------------------------------------------------------
367-- Stream of Arrays IO
368-------------------------------------------------------------------------------
370-------------------------------------------------------------------------------
372-------------------------------------------------------------------------------
374-- | Write a stream of arrays to a handle.
377{-# INLINE fromChunks #-}
378fromChunks :: (MonadIO m, Storable a)
379 => Handle -> Stream m (Array a) -> m ()
380fromChunks h m = S.mapM_ (liftIO . writeArray h) m
382-- | @fromChunksWithBufferOf bufsize handle stream@ writes a stream of arrays
383-- to @handle@ after coalescing the adjacent arrays in chunks of @bufsize@.
384-- The chunk size is only a maximum and the actual writes could be smaller as
385-- we do not split the arrays to fit exactly to the specified size.
388{-# INLINE fromChunksWithBufferOf #-}
389fromChunksWithBufferOf :: (MonadIO m, Storable a)
390 => Int -> Handle -> Stream m (Array a) -> m ()
391fromChunksWithBufferOf n h xs = fromChunks h $ AS.compact n xs
393-- | @fromStreamWithBufferOf bufsize handle stream@ writes @stream@ to @handle@
394-- in chunks of @bufsize@. A write is performed to the IO device as soon as we
395-- collect the required input size.
398{-# INLINE fromStreamWithBufferOf #-}
399fromStreamWithBufferOf :: MonadIO m => Int -> Handle -> Stream m Word8 -> m ()
400fromStreamWithBufferOf n h m = fromChunks h $ S.arraysOf n m
401-- fromStreamWithBufferOf n h m = fromChunks h $ AS.arraysOf n m
403-- > write = 'writeWithBufferOf' A.defaultChunkSize
405-- | Write a byte stream to a file handle. Accumulates the input in chunks of
406-- up to 'Streamly.Internal.Data.Array.Type.defaultChunkSize' before writing.
408-- NOTE: This may perform better than the 'write' fold, you can try this if you
409-- need some extra perf boost.
412{-# INLINE fromStream #-}
413fromStream :: MonadIO m => Handle -> Stream m Word8 -> m ()
414fromStream = fromStreamWithBufferOf defaultChunkSize
416-- | Write a stream of arrays to a handle. Each array in the stream is written
417-- to the device as a separate IO request.
420{-# INLINE writeChunks #-}
421writeChunks :: (MonadIO m, Storable a) => Handle -> Fold m (Array a) ()
422writeChunks h = FL.drainBy (liftIO . writeArray h)
424-- | @writeChunksWithBufferOf bufsize handle@ writes a stream of arrays
425-- to @handle@ after coalescing the adjacent arrays in chunks of @bufsize@.
426-- We never split an array, if a single array is bigger than the specified size
427-- it emitted as it is. Multiple arrays are coalesed as long as the total size
428-- remains below the specified size.
431{-# INLINE writeChunksWithBufferOf #-}
432writeChunksWithBufferOf :: (MonadIO m, Storable a)
433 => Int -> Handle -> Fold m (Array a) ()
434writeChunksWithBufferOf n h = lpackArraysChunksOf n (writeChunks h)
436-- GHC buffer size dEFAULT_FD_BUFFER_SIZE=8192 bytes.
439-- Note that if you use a chunk size less than 8K (GHC's default buffer
440-- size) then you are advised to use 'NOBuffering' mode on the 'Handle' in case you
441-- do not want buffering to occur at GHC level as well. Same thing applies to
444-- | @writeWithBufferOf reqSize handle@ writes the input stream to @handle@.
445-- Bytes in the input stream are collected into a buffer until we have a chunk
446-- of @reqSize@ and then written to the IO device.
449{-# INLINE writeWithBufferOf #-}
450writeWithBufferOf :: MonadIO m => Int -> Handle -> Fold m Word8 ()
451writeWithBufferOf n h = FL.chunksOf n (writeNUnsafe n) (writeChunks h)
453-- > write = 'writeWithBufferOf' A.defaultChunkSize
455-- | Write a byte stream to a file handle. Accumulates the input in chunks of
456-- up to 'Streamly.Internal.Data.Array.Type.defaultChunkSize' before writing
461write :: MonadIO m => Handle -> Fold m Word8 ()
462write = writeWithBufferOf defaultChunkSize
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list