299e6ef · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Add idiomatic definitions in docs

299e6efHarendra Kumar5 years ago

src/Streamly/Internal/FileSystem/File.hs+6−6

compare with master

unifiedsplit

@@ -71,7 +71,7 @@ module Streamly.Internal.FileSystem.File

71 , fromBytesWithBufferOf

73 -- -- * Array Write

74− , writeArray

74+ , putChunk

75 , writeChunks

76 , fromChunks

@@ -183,16 +183,16 @@ usingFile2 = UF.bracket before after

183-- | Write an array to a file. Overwrites the file if it exists.

184--

185-- @since 0.7.0

186−{-# INLINABLE writeArray #-}

186+{-# INLINABLE putChunk #-}

187−writeArray :: Storable a => FilePath -> Array a -> IO ()

187+putChunk :: Storable a => FilePath -> Array a -> IO ()

188−writeArray file arr = SIO.withFile file WriteMode (`FH.writeArray` arr)

188+putChunk file arr = SIO.withFile file WriteMode (`FH.putChunk` arr)

189

190-- | append an array to a file.

191--

192-- @since 0.7.0

193{-# INLINABLE appendArray #-}

194appendArray :: Storable a => FilePath -> Array a -> IO ()

195−appendArray file arr = SIO.withFile file AppendMode (`FH.writeArray` arr)

195+appendArray file arr = SIO.withFile file AppendMode (`FH.putChunk` arr)

196

197-------------------------------------------------------------------------------

198-- Stream of Arrays IO

@@ -298,7 +298,7 @@ readShared = undefined

298fromChunksMode :: (MonadAsync m, MonadCatch m, Storable a)

299 => IOMode -> FilePath -> SerialT m (Array a) -> m ()

300fromChunksMode mode file xs = S.drain $

301− withFile file mode (\h -> S.mapM (liftIO . FH.writeArray h) xs)

301+ withFile file mode (\h -> S.mapM (FH.putChunk h) xs)

302

303-- | Write a stream of arrays to a file. Overwrites the file if it exists.

304--

src/Streamly/Internal/FileSystem/Handle.hs+79−27

@@ -8,6 +8,11 @@

8-- Maintainer : streamly@composewell.com

9-- Stability : experimental

10-- Portability : GHC

11+--

12+-- The fundamental singleton APIs are 'getChunk' and 'putChunk' and the

13+-- fundamental stream APIs built on top of those are 'readChunksWithBufferOf'

14+-- and 'writeChunks'. Rest of this module is just combinatorial programming

15+-- using these.

12module Streamly.Internal.FileSystem.Handle

17module Streamly.Internal.FileSystem.Handle

13 (

18 (

@@ -22,7 +27,7 @@ module Streamly.Internal.FileSystem.Handle

22 , toBytesWithBufferOf

27 , toBytesWithBufferOf

24 -- -- * Array Read

29 -- -- * Array Read

25− -- , readArrayUpto

30+ , getChunk

26 -- , readArrayOf

31 -- , readArrayOf

27 , readChunks

32 , readChunks

28 , readChunksWithBufferOf

33 , readChunksWithBufferOf

@@ -45,7 +50,7 @@ module Streamly.Internal.FileSystem.Handle

45 , putBytesWithBufferOf

50 , putBytesWithBufferOf

47 -- -- * Array Write

52 -- -- * Array Write

48− , writeArray

53+ , putChunk

49 , writeChunks

54 , writeChunks

50 , writeChunksWithBufferOf

55 , writeChunksWithBufferOf

@@ -75,8 +80,8 @@ module Streamly.Internal.FileSystem.Handle

76 -- , readChunksFrom

81 -- , readChunksFrom

77 -- , readChunksFromTo

82 -- , readChunksFromTo

78− -- , readChunksFromToWithBufferOf

83+ -- , readChunksFromToWith

79− -- , readChunksFromThenToWithBufferOf

84+ -- , readChunksFromThenToWith

81 -- , writeIndex

86 -- , writeIndex

82 -- , writeFrom -- start writing at the given position

87 -- , writeFrom -- start writing at the given position

@@ -88,8 +93,8 @@ module Streamly.Internal.FileSystem.Handle

88 --

93 --

89 -- , writeChunksFrom

94 -- , writeChunksFrom

90 -- , writeChunksFromTo

95 -- , writeChunksFromTo

91− -- , writeChunksFromToWithBufferOf

96+ -- , writeChunksFromToWith

92− -- , writeChunksFromThenToWithBufferOf

97+ -- , writeChunksFromThenToWith

93 )

98 )

94where

99where

100

@@ -124,6 +129,18 @@ import qualified Streamly.Internal.Data.Stream.IsStream as S

124import qualified Streamly.Data.Array.Foreign as A

129import qualified Streamly.Data.Array.Foreign as A

125import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

130import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

126

131

132+-- $setup

133+-- >>> import qualified Streamly.Data.Array.Foreign as Array

134+-- >>> import qualified Streamly.Data.Fold as Fold

135+-- >>> import qualified Streamly.Data.Unfold as Unfold

136+-- >>> import qualified Streamly.FileSystem.Handle as Handle

137+-- >>> import qualified Streamly.Prelude as Stream

138+--

139+-- >>> import qualified Streamly.Internal.Data.Stream.IsStream as Stream

140+-- >>> import qualified Streamly.Internal.Data.Unfold as Unfold (supplyFirst)

141+-- >>> import qualified Streamly.Internal.FileSystem.Handle as Handle

142+-- >>> import qualified Streamly.Internal.System.IO as IO (defaultChunkSize)

143+

127-------------------------------------------------------------------------------

144-------------------------------------------------------------------------------

128-- References

145-- References

129-------------------------------------------------------------------------------

146-------------------------------------------------------------------------------

@@ -145,9 +162,9 @@ import qualified Streamly.Internal.Data.Stream.StreamD.Type as D

145-- handle it blocks until some data becomes available. If data is available

162-- handle it blocks until some data becomes available. If data is available

146-- then it immediately returns that data without blocking. It reads a maximum

163-- then it immediately returns that data without blocking. It reads a maximum

147-- of up to the size requested.

164-- of up to the size requested.

148−{-# INLINABLE readArrayUpto #-}

165+{-# INLINABLE getChunk #-}

149−readArrayUpto :: Int -> Handle -> IO (Array Word8)

166+getChunk :: Int -> Handle -> IO (Array Word8)

150−readArrayUpto size h = do

167+getChunk size h = do

151 ptr <- mallocPlainForeignPtrBytes size

168 ptr <- mallocPlainForeignPtrBytes size

152 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))

169 -- ptr <- mallocPlainForeignPtrAlignedBytes size (alignment (undefined :: Word8))

153 withForeignPtr ptr $ \p -> do

170 withForeignPtr ptr $ \p -> do

@@ -171,7 +188,7 @@ _toChunksWithBufferOf size h = go

171 where

188 where

172 -- XXX use cons/nil instead

189 -- XXX use cons/nil instead

173 go = mkStream $ \_ yld _ stp -> do

190 go = mkStream $ \_ yld _ stp -> do

174− arr <- liftIO $ readArrayUpto size h

191+ arr <- liftIO $ getChunk size h

175 if A.length arr == 0

192 if A.length arr == 0

176 then stp

193 then stp

177 else yld arr go

194 else yld arr go

@@ -180,14 +197,17 @@ _toChunksWithBufferOf size h = go

180-- handle @handle@. The maximum size of a single array is limited to @size@.

197-- handle @handle@. The maximum size of a single array is limited to @size@.

181-- The actual size read may be less than or equal to @size@.

198-- The actual size read may be less than or equal to @size@.

182--

199--

200+-- >>> toChunksWithBufferOf size h = Stream.unfold Handle.readChunksWithBufferOf (size, h)

201+--

183-- @since 0.7.0

202-- @since 0.7.0

184{-# INLINE_NORMAL toChunksWithBufferOf #-}

203{-# INLINE_NORMAL toChunksWithBufferOf #-}

185−toChunksWithBufferOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m (Array Word8)

204+toChunksWithBufferOf :: (IsStream t, MonadIO m) =>

205+ Int -> Handle -> t m (Array Word8)

186toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ())

206toChunksWithBufferOf size h = D.fromStreamD (D.Stream step ())

187 where

207 where

188 {-# INLINE_LATE step #-}

208 {-# INLINE_LATE step #-}

189 step _ _ = do

209 step _ _ = do

190− arr <- liftIO $ readArrayUpto size h

210+ arr <- liftIO $ getChunk size h

191 return $

211 return $

192 case A.length arr of

212 case A.length arr of

193 0 -> D.Stop

213 0 -> D.Stop

@@ -205,7 +225,7 @@ readChunksWithBufferOf = Unfold step return

205 where

225 where

206 {-# INLINE_LATE step #-}

226 {-# INLINE_LATE step #-}

207 step (size, h) = do

227 step (size, h) = do

208− arr <- liftIO $ readArrayUpto size h

228+ arr <- liftIO $ getChunk size h

209 return $

229 return $

210 case A.length arr of

230 case A.length arr of

211 0 -> D.Stop

231 0 -> D.Stop

@@ -218,7 +238,7 @@ readChunksWithBufferOf = Unfold step return

218-- @defaultChunkSize@. The actual size read may be less than or equal to

238-- @defaultChunkSize@. The actual size read may be less than or equal to

219-- @defaultChunkSize@.

239-- @defaultChunkSize@.

220--

240--

221−-- > toChunks = toChunksWithBufferOf defaultChunkSize

241+-- >>> toChunks = Handle.toChunksWithBufferOf IO.defaultChunkSize

222--

242--

223-- @since 0.7.0

243-- @since 0.7.0

224{-# INLINE toChunks #-}

244{-# INLINE toChunks #-}

@@ -231,6 +251,8 @@ toChunks = toChunksWithBufferOf defaultChunkSize

231-- size of arrays in the resulting stream are therefore less than or equal to

251-- size of arrays in the resulting stream are therefore less than or equal to

232-- 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize'.

252-- 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize'.

233--

253--

254+-- >>> readChunks = Unfold.supplyFirst IO.defaultChunkSize Handle.readChunksWithBufferOf

255+--

234-- @since 0.7.0

256-- @since 0.7.0

235{-# INLINE readChunks #-}

257{-# INLINE readChunks #-}

236readChunks :: MonadIO m => Unfold m Handle (Array Word8)

258readChunks :: MonadIO m => Unfold m Handle (Array Word8)

@@ -247,6 +269,8 @@ readChunks = UF.supplyFirst defaultChunkSize readChunksWithBufferOf

247-- | Unfolds the tuple @(bufsize, handle)@ into a byte stream, read requests

269-- | Unfolds the tuple @(bufsize, handle)@ into a byte stream, read requests

248-- to the IO device are performed using buffers of @bufsize@.

270-- to the IO device are performed using buffers of @bufsize@.

249--

271--

272+-- >>> readWithBufferOf = Unfold.many Handle.readChunksWithBufferOf Array.read

273+--

250-- @since 0.7.0

274-- @since 0.7.0

251{-# INLINE readWithBufferOf #-}

275{-# INLINE readWithBufferOf #-}

252readWithBufferOf :: MonadIO m => Unfold m (Int, Handle) Word8

276readWithBufferOf :: MonadIO m => Unfold m (Int, Handle) Word8

@@ -255,10 +279,12 @@ readWithBufferOf = UF.many readChunksWithBufferOf A.read

255-- | @toBytesWithBufferOf bufsize handle@ reads a byte stream from a file

279-- | @toBytesWithBufferOf bufsize handle@ reads a byte stream from a file

256-- handle, reads are performed in chunks of up to @bufsize@.

280-- handle, reads are performed in chunks of up to @bufsize@.

257--

281--

282+-- >>> toBytesWithBufferOf size h = Stream.unfoldMany Array.read $ Handle.toChunksWithBufferOf size h

283+--

258-- /Pre-release/

284-- /Pre-release/

259{-# INLINE toBytesWithBufferOf #-}

285{-# INLINE toBytesWithBufferOf #-}

260toBytesWithBufferOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8

286toBytesWithBufferOf :: (IsStream t, MonadIO m) => Int -> Handle -> t m Word8

261−toBytesWithBufferOf chunkSize h = AS.concat $ toChunksWithBufferOf chunkSize h

287+toBytesWithBufferOf size h = AS.concat $ toChunksWithBufferOf size h

262

288

263-- TODO

289-- TODO

264-- Generate a stream of elements of the given type from a file 'Handle'.

290-- Generate a stream of elements of the given type from a file 'Handle'.

@@ -268,13 +294,17 @@ toBytesWithBufferOf chunkSize h = AS.concat $ toChunksWithBufferOf chunkSize h

268-- performed in sizes of

294-- performed in sizes of

269-- 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize'.

295-- 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize'.

270--

296--

297+-- >>> read = Unfold.many Handle.readChunks Array.read

298+--

271-- @since 0.7.0

299-- @since 0.7.0

272{-# INLINE read #-}

300{-# INLINE read #-}

273read :: MonadIO m => Unfold m Handle Word8

301read :: MonadIO m => Unfold m Handle Word8

274−read = UF.supplyFirst defaultChunkSize readWithBufferOf

302+read = UF.many readChunks A.read

275

303

276-- | Generate a byte stream from a file 'Handle'.

304-- | Generate a byte stream from a file 'Handle'.

277--

305--

306+-- >>> toBytes h = Stream.unfoldMany Array.read $ Handle.toChunks h

307+--

278-- /Pre-release/

308-- /Pre-release/

279{-# INLINE toBytes #-}

309{-# INLINE toBytes #-}

280toBytes :: (IsStream t, MonadIO m) => Handle -> t m Word8

310toBytes :: (IsStream t, MonadIO m) => Handle -> t m Word8

@@ -291,11 +321,14 @@ toBytes = AS.concat . toChunks

291-- | Write an 'Array' to a file handle.

321-- | Write an 'Array' to a file handle.

292--

322--

293-- @since 0.7.0

323-- @since 0.7.0

294−{-# INLINABLE writeArray #-}

324+{-# INLINABLE putChunk #-}

295−writeArray :: Storable a => Handle -> Array a -> IO ()

325+putChunk :: (MonadIO m, Storable a) => Handle -> Array a -> m ()

296−writeArray _ arr | A.length arr == 0 = return ()

326+putChunk _ arr | A.length arr == 0 = return ()

327+putChunk h Array{..} =

297−writeArray h Array{..} = unsafeWithForeignPtr aStart $ \p -> hPutBuf h p aLen

328+ liftIO $ unsafeWithForeignPtr aStart $ \p -> hPutBuf h p aLen

329+

298 where

330 where

331+

299 aLen =

332 aLen =

300 let p = unsafeForeignPtrToPtr aStart

333 let p = unsafeForeignPtrToPtr aStart

301 in aEnd `minusPtr` p

334 in aEnd `minusPtr` p

@@ -311,12 +344,18 @@ writeArray h Array{..} = unsafeWithForeignPtr aStart $ \p -> hPutBuf h p aLen

311-- XXX use an unfold to fromObjects or fromUnfold so that we can put any object

344-- XXX use an unfold to fromObjects or fromUnfold so that we can put any object

312-- | Write a stream of arrays to a handle.

345-- | Write a stream of arrays to a handle.

313--

346--

347+-- >>> putChunks h = Stream.mapM_ (Handle.putChunk h)

348+--

314-- @since 0.7.0

349-- @since 0.7.0

315{-# INLINE putChunks #-}

350{-# INLINE putChunks #-}

316putChunks :: (MonadIO m, Storable a)

351putChunks :: (MonadIO m, Storable a)

317 => Handle -> SerialT m (Array a) -> m ()

352 => Handle -> SerialT m (Array a) -> m ()

318−putChunks h = S.mapM_ (liftIO . writeArray h)

353+putChunks h = S.mapM_ (putChunk h)

319

354

355+-- XXX AS.compact can be written idiomatically in terms of foldMany, just like

356+-- AS.concat is written in terms of foldMany. Once that is done we can write

357+-- idiomatic def in the docs.

358+--

320-- | @putChunksWithBufferOf bufsize handle stream@ writes a stream of arrays

359-- | @putChunksWithBufferOf bufsize handle stream@ writes a stream of arrays

321-- to @handle@ after coalescing the adjacent arrays in chunks of @bufsize@.

360-- to @handle@ after coalescing the adjacent arrays in chunks of @bufsize@.

322-- The chunk size is only a maximum and the actual writes could be smaller as

361-- The chunk size is only a maximum and the actual writes could be smaller as

@@ -332,20 +371,22 @@ putChunksWithBufferOf n h xs = putChunks h $ AS.compact n xs

332-- in chunks of @bufsize@. A write is performed to the IO device as soon as we

371-- in chunks of @bufsize@. A write is performed to the IO device as soon as we

333-- collect the required input size.

372-- collect the required input size.

334--

373--

374+-- >>> putBytesWithBufferOf n h m = Handle.putChunks h $ Stream.arraysOf n m

375+--

335-- @since 0.7.0

376-- @since 0.7.0

336{-# INLINE putBytesWithBufferOf #-}

377{-# INLINE putBytesWithBufferOf #-}

337putBytesWithBufferOf :: MonadIO m => Int -> Handle -> SerialT m Word8 -> m ()

378putBytesWithBufferOf :: MonadIO m => Int -> Handle -> SerialT m Word8 -> m ()

338putBytesWithBufferOf n h m = putChunks h $ S.arraysOf n m

379putBytesWithBufferOf n h m = putChunks h $ S.arraysOf n m

339-- putBytesWithBufferOf n h m = putChunks h $ AS.arraysOf n m

380-- putBytesWithBufferOf n h m = putChunks h $ AS.arraysOf n m

340

381

341−-- > write = 'writeWithBufferOf' A.defaultChunkSize

342−--

343-- | Write a byte stream to a file handle. Accumulates the input in chunks of

382-- | Write a byte stream to a file handle. Accumulates the input in chunks of

344-- up to 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize' before writing.

383-- up to 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize' before writing.

345--

384--

346-- NOTE: This may perform better than the 'write' fold, you can try this if you

385-- NOTE: This may perform better than the 'write' fold, you can try this if you

347-- need some extra perf boost.

386-- need some extra perf boost.

348--

387--

388+-- >>> putBytes = Handle.putBytesWithBufferOf IO.defaultChunkSize

389+--

349-- @since 0.7.0

390-- @since 0.7.0

350{-# INLINE putBytes #-}

391{-# INLINE putBytes #-}

351putBytes :: MonadIO m => Handle -> SerialT m Word8 -> m ()

392putBytes :: MonadIO m => Handle -> SerialT m Word8 -> m ()

@@ -354,15 +395,22 @@ putBytes = putBytesWithBufferOf defaultChunkSize

354-- | Write a stream of arrays to a handle. Each array in the stream is written

395-- | Write a stream of arrays to a handle. Each array in the stream is written

355-- to the device as a separate IO request.

396-- to the device as a separate IO request.

356--

397--

398+-- writeChunks h = Fold.drainBy (Handle.putChunk h)

399+--

357-- @since 0.7.0

400-- @since 0.7.0

358{-# INLINE writeChunks #-}

401{-# INLINE writeChunks #-}

359writeChunks :: (MonadIO m, Storable a) => Handle -> Fold m (Array a) ()

402writeChunks :: (MonadIO m, Storable a) => Handle -> Fold m (Array a) ()

360−writeChunks h = FL.drainBy (liftIO . writeArray h)

403+writeChunks h = FL.drainBy (putChunk h)

361

404

362{-# INLINE writeChunks2 #-}

405{-# INLINE writeChunks2 #-}

363writeChunks2 :: (MonadIO m, Storable a) => Fold2 m Handle (Array a) ()

406writeChunks2 :: (MonadIO m, Storable a) => Fold2 m Handle (Array a) ()

407+writeChunks2 =

364−writeChunks2 = Fold2 (\h arr -> liftIO $ writeArray h arr >> return h) return (\_ -> return ())

408+ Fold2 (\h arr -> liftIO $ putChunk h arr >> return h)

409+ return

410+ (\_ -> return ())

365

411

412+-- XXX lpackArraysChunksOf should be written idiomatically

413+--

366-- | @writeChunksWithBufferOf bufsize handle@ writes a stream of arrays

414-- | @writeChunksWithBufferOf bufsize handle@ writes a stream of arrays

367-- to @handle@ after coalescing the adjacent arrays in chunks of @bufsize@.

415-- to @handle@ after coalescing the adjacent arrays in chunks of @bufsize@.

368-- We never split an array, if a single array is bigger than the specified size

416-- We never split an array, if a single array is bigger than the specified size

@@ -383,10 +431,14 @@ writeChunksWithBufferOf n h = lpackArraysChunksOf n (writeChunks h)

383-- do not want buffering to occur at GHC level as well. Same thing applies to

431-- do not want buffering to occur at GHC level as well. Same thing applies to

384-- writes as well.

432-- writes as well.

385

433

434+-- XXX Maybe we should have a Fold.arraysOf like we have Stream.arraysOf

435+--

386-- | @writeWithBufferOf reqSize handle@ writes the input stream to @handle@.

436-- | @writeWithBufferOf reqSize handle@ writes the input stream to @handle@.

387-- Bytes in the input stream are collected into a buffer until we have a chunk

437-- Bytes in the input stream are collected into a buffer until we have a chunk

388-- of @reqSize@ and then written to the IO device.

438-- of @reqSize@ and then written to the IO device.

389--

439--

440+-- >>> writeWithBufferOf n h = Fold.chunksOf n (Array.writeNUnsafe n) (Handle.writeChunks h)

441+--

390-- @since 0.7.0

442-- @since 0.7.0

391{-# INLINE writeWithBufferOf #-}

443{-# INLINE writeWithBufferOf #-}

392writeWithBufferOf :: MonadIO m => Int -> Handle -> Fold m Word8 ()

444writeWithBufferOf :: MonadIO m => Int -> Handle -> Fold m Word8 ()

@@ -396,12 +448,12 @@ writeWithBufferOf n h = FL.chunksOf n (writeNUnsafe n) (writeChunks h)

396writeWithBufferOf2 :: MonadIO m => Int -> Fold2 m Handle Word8 ()

448writeWithBufferOf2 :: MonadIO m => Int -> Fold2 m Handle Word8 ()

397writeWithBufferOf2 n = FL.chunksOf2 n (writeNUnsafe n) writeChunks2

449writeWithBufferOf2 n = FL.chunksOf2 n (writeNUnsafe n) writeChunks2

398

450

399−-- > write = 'writeWithBufferOf' A.defaultChunkSize

400−--

401-- | Write a byte stream to a file handle. Accumulates the input in chunks of

451-- | Write a byte stream to a file handle. Accumulates the input in chunks of

402-- up to 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize' before writing

452-- up to 'Streamly.Internal.Data.Array.Foreign.Type.defaultChunkSize' before writing

403-- to the IO device.

453-- to the IO device.

404--

454--

455+-- >>> write = Handle.writeWithBufferOf IO.defaultChunkSize

456+--

405-- @since 0.7.0

457-- @since 0.7.0

406{-# INLINE write #-}

458{-# INLINE write #-}

407write :: MonadIO m => Handle -> Fold m Word8 ()

459write :: MonadIO m => Handle -> Fold m Word8 ()