6d9ed26 · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Read filesystem directories as streams

6d9ed26Ranjeet Kumar Ranjan4 years ago

core/src/Streamly/Internal/FileSystem/Dir.hs+65−19

compare with master

unifiedsplit

@@ -82,15 +82,22 @@ module Streamly.Internal.FileSystem.Dir

82 )

83where

85+import Control.Monad.Catch (MonadCatch)

85import Control.Monad.IO.Class (MonadIO(..))

86import Control.Monad.IO.Class (MonadIO(..))

86import Data.Bifunctor (bimap)

87import Data.Bifunctor (bimap)

87import Data.Either (isRight, isLeft, fromLeft, fromRight)

88import Data.Either (isRight, isLeft, fromLeft, fromRight)

89+import Data.Function ((&))

88import Streamly.Data.Stream (Stream)

90import Streamly.Data.Stream (Stream)

91+import Streamly.Internal.Data.Unfold (Step(..))

89import Streamly.Internal.Data.Unfold.Type (Unfold(..))

92import Streamly.Internal.Data.Unfold.Type (Unfold(..))

90import System.FilePath ((</>))

93import System.FilePath ((</>))

91−

94+#if (defined linux_HOST_OS) || (defined darwin_HOST_OS)

95+import System.Posix (openDirStream, readDirStream, closeDirStream)

96+#elif defined(mingw32_HOST_OS)

97+import qualified System.Win32 as Win32

98+#endif

92import qualified Streamly.Data.Unfold as UF

99import qualified Streamly.Data.Unfold as UF

93−import qualified Streamly.Internal.Data.Unfold as UF (mapM2)

100+import qualified Streamly.Internal.Data.Unfold as UF (mapM2, bracketIO)

94import qualified Streamly.Data.Stream as S

101import qualified Streamly.Data.Stream as S

95import qualified System.Directory as Dir

102import qualified System.Directory as Dir

103

@@ -236,11 +243,50 @@ toStreamWithBufferOf chunkSize h = AS.concat $ toChunksWithBufferOf chunkSize h

236-- /Internal/

243-- /Internal/

237--

244--

238{-# INLINE reader #-}

245{-# INLINE reader #-}

239−reader :: MonadIO m => Unfold m FilePath FilePath

246+reader :: (MonadIO m, MonadCatch m) => Unfold m FilePath FilePath

240reader =

247reader =

241− -- XXX use proper streaming read of the dir

248+#if (defined linux_HOST_OS) || (defined darwin_HOST_OS)

249+ UF.bracketIO openDirStream closeDirStream childOf

250+ & UF.filter (\x -> x /= "." && x /= "..")

251+

252+ where

253+

254+ {-# INLINABLE childOf #-}

255+ childOf = Unfold step inject

256+

257+ where

258+

259+ inject = return

260+

261+ step strm = do

262+ file <- liftIO $ readDirStream strm

263+ case file of

264+ [] -> return Stop

265+ _ -> return $ Yield file strm

266+#elif defined(mingw32_HOST_OS)

267+ UF.bracketIO

268+ Win32.findFirstFile

269+ (\(h, _) -> liftIO $ Win32.findClose h)

270+ childOf

242− UF.filter (\x -> x /= "." && x /= "..")

271+ & UF.filter (\x -> x /= "." && x /= "..")

243− $ UF.lmapM (liftIO . Dir.getDirectoryContents) UF.fromList

272+

273+ where

274+

275+ {-# INLINABLE childOf #-}

276+ childOf = Unfold step inject

277+

278+ where

279+

280+ inject = return

281+

282+ step (h, fdat) = do

283+ more <- liftIO $ Win32.findNextFile h fdat

284+ if more

285+ then do

286+ file <- liftIO $ Win32.getFindDataFileName fdat

287+ return $ Yield file (h, fdat)

288+ else return Stop

289+#endif

244

290

245-- XXX We can use a more general mechanism to filter the contents of a

291-- XXX We can use a more general mechanism to filter the contents of a

246-- directory. We can just stat each child and pass on the stat information. We

292-- directory. We can just stat each child and pass on the stat information. We

@@ -253,7 +299,7 @@ reader =

253-- /Internal/

299-- /Internal/

254--

300--

255{-# INLINE eitherReader #-}

301{-# INLINE eitherReader #-}

256−eitherReader :: MonadIO m => Unfold m FilePath (Either FilePath FilePath)

302+eitherReader :: (MonadIO m, MonadCatch m) => Unfold m FilePath (Either FilePath FilePath)

257eitherReader = UF.mapM2 classify reader

303eitherReader = UF.mapM2 classify reader

258

304

259 where

305 where

@@ -263,7 +309,7 @@ eitherReader = UF.mapM2 classify reader

263 return $ if r then Left x else Right x

309 return $ if r then Left x else Right x

264

310

265{-# INLINE eitherReaderPaths #-}

311{-# INLINE eitherReaderPaths #-}

266−eitherReaderPaths :: MonadIO m => Unfold m FilePath (Either FilePath FilePath)

312+eitherReaderPaths ::(MonadIO m, MonadCatch m) => Unfold m FilePath (Either FilePath FilePath)

267eitherReaderPaths =

313eitherReaderPaths =

268 UF.mapM2 (\dir -> return . bimap (dir </>) (dir </>)) eitherReader

314 UF.mapM2 (\dir -> return . bimap (dir </>) (dir </>)) eitherReader

269

315

@@ -273,7 +319,7 @@ eitherReaderPaths =

273-- /Internal/

319-- /Internal/

274--

320--

275{-# INLINE fileReader #-}

321{-# INLINE fileReader #-}

276−fileReader :: MonadIO m => Unfold m FilePath FilePath

322+fileReader :: (MonadIO m, MonadCatch m) => Unfold m FilePath FilePath

277fileReader = fmap (fromRight undefined) $ UF.filter isRight eitherReader

323fileReader = fmap (fromRight undefined) $ UF.filter isRight eitherReader

278

324

279-- | Read directories only. Filter out "." and ".." entries.

325-- | Read directories only. Filter out "." and ".." entries.

@@ -281,19 +327,19 @@ fileReader = fmap (fromRight undefined) $ UF.filter isRight eitherReader

281-- /Internal/

327-- /Internal/

282--

328--

283{-# INLINE dirReader #-}

329{-# INLINE dirReader #-}

284−dirReader :: MonadIO m => Unfold m FilePath FilePath

330+dirReader :: (MonadIO m, MonadCatch m) => Unfold m FilePath FilePath

285dirReader = fmap (fromLeft undefined) $ UF.filter isLeft eitherReader

331dirReader = fmap (fromLeft undefined) $ UF.filter isLeft eitherReader

286

332

287-- | Raw read of a directory.

333-- | Raw read of a directory.

288--

334--

289-- /Pre-release/

335-- /Pre-release/

290{-# INLINE read #-}

336{-# INLINE read #-}

291−read :: MonadIO m => FilePath -> Stream m FilePath

337+read :: (MonadIO m, MonadCatch m) => FilePath -> Stream m FilePath

292read = S.unfold reader

338read = S.unfold reader

293

339

294{-# DEPRECATED toStream "Please use 'read' instead" #-}

340{-# DEPRECATED toStream "Please use 'read' instead" #-}

295{-# INLINE toStream #-}

341{-# INLINE toStream #-}

296−toStream :: MonadIO m => String -> Stream m String

342+toStream :: (MonadIO m, MonadCatch m) => String -> Stream m String

297toStream = read

343toStream = read

298

344

299-- | Read directories as Left and files as Right. Filter out "." and ".."

345-- | Read directories as Left and files as Right. Filter out "." and ".."

@@ -301,18 +347,18 @@ toStream = read

301--

347--

302-- /Pre-release/

348-- /Pre-release/

303{-# INLINE readEither #-}

349{-# INLINE readEither #-}

304−readEither :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)

350+readEither :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Either FilePath FilePath)

305readEither = S.unfold eitherReader

351readEither = S.unfold eitherReader

306

352

307-- | Like 'readEither' but prefix the names of the files and directories with

353-- | Like 'readEither' but prefix the names of the files and directories with

308-- the supplied directory path.

354-- the supplied directory path.

309{-# INLINE readEitherPaths #-}

355{-# INLINE readEitherPaths #-}

310−readEitherPaths :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)

356+readEitherPaths :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Either FilePath FilePath)

311readEitherPaths dir = fmap (bimap (dir </>) (dir </>)) $ readEither dir

357readEitherPaths dir = fmap (bimap (dir </>) (dir </>)) $ readEither dir

312

358

313{-# DEPRECATED toEither "Please use 'readEither' instead" #-}

359{-# DEPRECATED toEither "Please use 'readEither' instead" #-}

314{-# INLINE toEither #-}

360{-# INLINE toEither #-}

315−toEither :: MonadIO m => FilePath -> Stream m (Either FilePath FilePath)

361+toEither :: (MonadIO m, MonadCatch m) => FilePath -> Stream m (Either FilePath FilePath)

316toEither = readEither

362toEither = readEither

317

363

318-- | Read files only.

364-- | Read files only.

@@ -320,12 +366,12 @@ toEither = readEither

320-- /Internal/

366-- /Internal/

321--

367--

322{-# INLINE readFiles #-}

368{-# INLINE readFiles #-}

323−readFiles :: MonadIO m => FilePath -> Stream m FilePath

369+readFiles :: (MonadIO m, MonadCatch m) => FilePath -> Stream m FilePath

324readFiles = S.unfold fileReader

370readFiles = S.unfold fileReader

325

371

326{-# DEPRECATED toFiles "Please use 'readFiles' instead" #-}

372{-# DEPRECATED toFiles "Please use 'readFiles' instead" #-}

327{-# INLINE toFiles #-}

373{-# INLINE toFiles #-}

328−toFiles :: MonadIO m => FilePath -> Stream m FilePath

374+toFiles :: (MonadIO m, MonadCatch m) => FilePath -> Stream m FilePath

329toFiles = readFiles

375toFiles = readFiles

330

376

331-- | Read directories only.

377-- | Read directories only.

@@ -333,12 +379,12 @@ toFiles = readFiles

333-- /Internal/

379-- /Internal/

334--

380--

335{-# INLINE readDirs #-}

381{-# INLINE readDirs #-}

336−readDirs :: MonadIO m => FilePath -> Stream m FilePath

382+readDirs :: (MonadIO m, MonadCatch m) => FilePath -> Stream m FilePath

337readDirs = S.unfold dirReader

383readDirs = S.unfold dirReader

338

384

339{-# DEPRECATED toDirs "Please use 'readDirs' instead" #-}

385{-# DEPRECATED toDirs "Please use 'readDirs' instead" #-}

340{-# INLINE toDirs #-}

386{-# INLINE toDirs #-}

341−toDirs :: MonadIO m => String -> Stream m String

387+toDirs :: (MonadIO m, MonadCatch m) => String -> Stream m String

342toDirs = readDirs

388toDirs = readDirs

343

389

344{-

390{-

core/streamly-core.cabal+6−1

@@ -139,7 +139,6 @@ common compile-options

139 if flag(use-folds)

140 cpp-options: -DUSE_FOLDS_EVERYWHERE

141

142−

143 ghc-options: -Weverything

142 ghc-options: -Weverything

144 -Wno-implicit-prelude

143 -Wno-implicit-prelude

145 -Wno-missing-deriving-strategies

144 -Wno-missing-deriving-strategies

@@ -470,3 +469,9 @@ library

470 , heaps >= 0.3 && < 0.5

469 , heaps >= 0.3 && < 0.5

471 if !flag(use-unliftio)

470 if !flag(use-unliftio)

472 build-depends: monad-control >= 1.0 && < 1.1

471 build-depends: monad-control >= 1.0 && < 1.1

472+

473+ if os(linux) || os (darwin)

474+ build-depends: unix >= 2.7.0 && < 2.9

475+

476+ if os(windows)

477+ build-depends: Win32 >= 2.6 && < 2.13