on master help sign in
Split Streams.hs into individual stream type files
commit8830c323a84f2625fbbfbe0fbd46ad03df8acee4 authorHarendra Kumar <harendra.kumar@gmail.com> dateFri 22 Jun 2018, 03:58 (8 years ago) parent9ad5b20
15 files changed+2591−2389
src/Streamly.hs+8−1 src/Streamly/CPS/Concurrent.hs−640 src/Streamly/CPS/Stream.hs−183 src/Streamly/Prelude.hs+9−72 src/Streamly/Streams.hs−1489 src/Streamly/Streams/Ahead.hs+373 src/Streamly/Streams/Async.hs+570 src/Streamly/Streams/CPS.hs+363 src/Streamly/Streams/Instances.hs+40 src/Streamly/Streams/Parallel.hs+372 src/Streamly/Streams/Prelude.hs+164 src/Streamly/Streams/SVar.hs+84 src/Streamly/Streams/Serial.hs+344 src/Streamly/Streams/Zip.hs+255 streamly.cabal+9−4
src/Streamly.hs+8−1
@@ -152,7 +152,14 @@ module Streamly
152152 )
153153where
154154
155−import Streamly.Streams
155+import Streamly.Streams.CPS hiding (runStream, serial)
156+import Streamly.Streams.Serial
157+import Streamly.Streams.Async
158+import Streamly.Streams.Ahead
159+import Streamly.Streams.Parallel
160+import Streamly.Streams.Zip
161+import Streamly.Streams.Prelude
162+import Streamly.SVar (MonadAsync)
156163import Data.Semigroup (Semigroup(..))
157164
158165-- $serial
src/Streamly/CPS/Concurrent.hsdeleted−640
@@ -1,640 +0,0 @@
1−{-# LANGUAGE CPP #-}
2−{-# LANGUAGE ConstraintKinds #-}
3−{-# LANGUAGE FlexibleContexts #-}
4−{-# LANGUAGE FlexibleInstances #-}
5−{-# LANGUAGE LambdaCase #-}
6−{-# LANGUAGE MagicHash #-}
7−{-# LANGUAGE MultiParamTypeClasses #-}
8−{-# LANGUAGE RankNTypes #-}
9−{-# LANGUAGE UnboxedTuples #-}
10−{-# LANGUAGE UndecidableInstances #-} -- XXX
11−
12−-- |
13−-- Module : Streamly.CPS.Concurrent
14−-- Copyright : (c) 2017 Harendra Kumar
15−--
16−-- License : BSD3
17−-- Maintainer : harendra.kumar@gmail.com
18−-- Stability : experimental
19−-- Portability : GHC
20−--
21−--
22−module Streamly.CPS.Concurrent
23− (
24− -- * Construction (monadic)
25− consMAhead
26− , consMAsync
27− , consMWAsync
28− , consMParallel
29−
30− -- * Semigroup Style Composition
31− , ahead
32− , async
33− , wAsync
34− , parallel
35−
36− -- * applications
37− , applyWith
38− , runWith
39−
40− -- * zip
41− , zipAsyncWith
42−
43− -- * Concurrent Stream Vars (SVars)
44− , fromStreamVar
45− , mkParallel
46−
47− )
48−where
49−
50−--
51−import Control.Concurrent.MVar (newEmptyMVar)
52−import Control.Monad.Catch (throwM)
53−import Control.Monad.IO.Class (MonadIO(..))
54−import Data.Atomics (atomicModifyIORefCAS_)
55−import Data.Concurrent.Queue.MichaelScott
56− (LinkedQueue, newQ, nullQ)
57−import Data.Functor (void)
58−import Data.Heap (Heap, Entry(..))
59−import Data.IORef (IORef, newIORef, readIORef)
60−import Data.Maybe (fromJust)
61−import GHC.IO (IO(..))
62−import Prelude hiding (repeat, zipWith)
63−
64−import qualified Data.Set as S
65−import qualified Data.Heap as H
66−
67−import Streamly.SVar
68−import Streamly.CPS.Stream
69−
70−-- MVar diagnostics has some overhead - around 5% on asyncly null benchmark, we
71−-- can keep it on in production to debug problems quickly if and when they
72−-- happen, but it may result in unexpected output when threads are left hanging
73−-- until they are GCed because the consumer went away.
74−
75−#ifdef DIAGNOSTICS
76−import Control.Monad (when)
77−import Data.IORef (writeIORef)
78−#endif
79−
80−-------------------------------------------------------------------------------
81−-- Async
82−-------------------------------------------------------------------------------
83−
84−{-# INLINE runStreamLIFO #-}
85−runStreamLIFO :: MonadIO m
86− => SVar Stream m a -> IORef [Stream m a] -> Stream m a -> m () -> m ()
87−runStreamLIFO sv q m stop = runStream m (Just sv) stop single yield
88− where
89− single a = do
90− res <- liftIO $ send sv (ChildYield a)
91− if res then stop else liftIO $ sendStop sv
92− yield a r = do
93− res <- liftIO $ send sv (ChildYield a)
94− if res
95− then (runStream r) (Just sv) stop single yield
96− else liftIO $ enqueueLIFO sv q r >> sendStop sv
97−
98−-------------------------------------------------------------------------------
99−-- WAsync
100−-------------------------------------------------------------------------------
101−
102−{-# INLINE runStreamFIFO #-}
103−runStreamFIFO :: MonadIO m
104− => SVar Stream m a -> LinkedQueue (Stream m a) -> Stream m a -> m () -> m ()
105−runStreamFIFO sv q m stop = runStream m (Just sv) stop single yield
106− where
107− single a = do
108− res <- liftIO $ send sv (ChildYield a)
109− if res then stop else liftIO $ sendStop sv
110− yield a r = do
111− res <- liftIO $ send sv (ChildYield a)
112− liftIO (enqueueFIFO sv q r)
113− if res then stop else liftIO $ sendStop sv
114−
115−-------------------------------------------------------------------------------
116−-- Parallel
117−-------------------------------------------------------------------------------
118−
119−{-# NOINLINE runOne #-}
120−runOne :: MonadIO m => SVar Stream m a -> Stream m a -> m ()
121−runOne sv m = (runStream m) (Just sv) stop single yield
122−
123− where
124−
125− stop = liftIO $ sendStop sv
126− sendit a = liftIO $ send sv (ChildYield a)
127− single a = sendit a >> stop
128− -- XXX there is no flow control in parallel case. We should perhaps use a
129− -- queue and queue it back on that and exit the thread when the outputQueue
130− -- overflows. Parallel is dangerous because it can accumulate unbounded
131− -- output in the buffer.
132− yield a r = void (sendit a) >> runOne sv r
133−
134−-------------------------------------------------------------------------------
135−-- Ahead
136−-------------------------------------------------------------------------------
137−
138−-- Lookahead streams can execute multiple tasks concurrently, ahead of time,
139−-- but always serve them in the same order as they appear in the stream. To
140−-- implement lookahead streams efficiently we assign a sequence number to each
141−-- task when the task is picked up for execution. When the task finishes, the
142−-- output is tagged with the same sequence number and we rearrange the outputs
143−-- in sequence based on that number.
144−--
145−-- To explain the mechanism imagine that the current task at the head of the
146−-- stream has a "token" to yield to the outputQueue. The ownership of the token
147−-- is determined by the current sequence number is maintained in outputHeap.
148−-- Sequence number is assigned when a task is queued. When a thread dequeues a
149−-- task it picks up the sequence number as well and when the output is ready it
150−-- uses the sequence number to queue the output to the outputQueue.
151−--
152−-- The thread with current sequence number sends the output directly to the
153−-- outputQueue. Other threads push the output to the outputHeap. When the task
154−-- being queued on the heap is a stream of many elements we evaluate only the
155−-- first element and keep the rest of the unevaluated computation in the heap.
156−-- When such a task gets the "token" for outputQueue it evaluates and directly
157−-- yields all the elements to the outputQueue without checking for the
158−-- "token".
159−--
160−-- Note that no two outputs in the heap can have the same sequence numbers and
161−-- therefore we do not need a stable heap. We have also separated the buffer
162−-- for the current task (outputQueue) and the pending tasks (outputHeap) so
163−-- that the pending tasks cannot interfere with the current task. Note that for
164−-- a single task just the outputQueue is enough and for the case of many
165−-- threads just a heap is good enough. However we balance between these two
166−-- cases, so that both are efficient.
167−--
168−-- For bigger streams it may make sense to have separate buffers for each
169−-- stream. However, for singleton streams this may become inefficient. However,
170−-- if we do not have separate buffers, then the streams that come later in
171−-- sequence may hog the buffer, hindering the streams that are ahead. For this
172−-- reason we have a single element buffer limitation for the streams being
173−-- executed in advance.
174−--
175−-- This scheme works pretty efficiently with less than 40% extra overhead
176−-- compared to the Async streams where we do not have any kind of sequencing of
177−-- the outputs. It is especially devised so that we are most efficient when we
178−-- have short tasks and need just a single thread. Also when a thread yields
179−-- many items it can hold lockfree access to the outputQueue and do it
180−-- efficiently.
181−--
182−-- XXX Maybe we can start the ahead threads at a lower cpu and IO priority so
183−-- that they do not hog the resources and hinder the progress of the threads in
184−-- front of them.
185−
186−-- Left associated ahead expressions are expensive. We start a new SVar for
187−-- each left associative expression. The queue is used only for right
188−-- associated expression, we queue the right expression and execute the left.
189−-- Thererefore the queue never has more than on item in it.
190−
191−workLoopAhead :: MonadIO m
192− => SVar Stream m a
193− -> IORef ([Stream m a], Int)
194− -> IORef (Heap (Entry Int (AheadHeapEntry Stream m a)) , Int)
195− -> m ()
196−workLoopAhead sv q heap = runHeap
197−
198− where
199−
200− toHeap seqNo ent = do
201− hp <- liftIO $ atomicModifyIORefCAS heap $ \(h, snum) ->
202− ((H.insert (Entry seqNo ent) h, snum), h)
203− if H.size hp <= maxHeap
204− then runHeap
205− else liftIO $ sendStop sv
206−
207− singleToHeap seqNo a = toHeap seqNo (AheadEntryPure a)
208− yieldToHeap seqNo a r = toHeap seqNo (AheadEntryStream (a `cons` r))
209−
210− singleOutput seqNo a = do
211− continue <- liftIO $ send sv (ChildYield a)
212− if continue
213− then runQueueToken seqNo
214− else liftIO $ do
215− atomicModifyIORefCAS_ heap $ \(h, _) -> (h, seqNo + 1)
216− sendStop sv
217−
218− yieldOutput seqNo a r = do
219− continue <- liftIO $ send sv (ChildYield a)
220− if continue
221− then (runStream r) (Just sv) (runQueueToken seqNo)
222− (singleOutput seqNo)
223− (yieldOutput seqNo)
224− else liftIO $ do
225− atomicModifyIORefCAS_ heap $ \(h, _) ->
226− (H.insert (Entry seqNo (AheadEntryStream r)) h, seqNo)
227− sendStop sv
228−
229− {-# INLINE runQueueToken #-}
230− runQueueToken prevSeqNo = do
231− work <- dequeueAhead q
232− case work of
233− Nothing -> do
234− liftIO $ atomicModifyIORefCAS_ heap $ \(h, _) ->
235− (h, prevSeqNo + 1)
236− runHeap
237− Just (m, seqNo) -> do
238− if seqNo == prevSeqNo + 1
239− then
240− (runStream m) (Just sv) (runQueueToken seqNo)
241− (singleOutput seqNo)
242− (yieldOutput seqNo)
243− else do
244− liftIO $ atomicModifyIORefCAS_ heap $ \(h, _) ->
245− (h, prevSeqNo + 1)
246− (runStream m) (Just sv) runHeap
247− (singleToHeap seqNo)
248− (yieldToHeap seqNo)
249− runQueueNoToken = do
250− work <- dequeueAhead q
251− case work of
252− Nothing -> runHeap
253− Just (m, seqNo) -> do
254− if seqNo == 0
255− then
256− (runStream m) (Just sv) (runQueueToken seqNo)
257− (singleOutput seqNo)
258− (yieldOutput seqNo)
259− else
260− (runStream m) (Just sv) runHeap
261− (singleToHeap seqNo)
262− (yieldToHeap seqNo)
263−
264− {-# NOINLINE runHeap #-}
265− runHeap = do
266−#ifdef DIAGNOSTICS
267− liftIO $ do
268− maxHp <- readIORef (maxHeapSize sv)
269− (hp, _) <- readIORef heap
270− when (H.size hp > maxHp) $ writeIORef (maxHeapSize sv) (H.size hp)
271−#endif
272− ent <- liftIO $ dequeueFromHeap heap
273− case ent of
274− Nothing -> do
275− done <- queueEmptyAhead q
276− if done
277− then liftIO $ sendStop sv
278− else runQueueNoToken
279− Just (Entry seqNo hent) -> do
280− case hent of
281− AheadEntryPure a -> singleOutput seqNo a
282− AheadEntryStream r ->
283− (runStream r) (Just sv) (runQueueToken seqNo)
284− (singleOutput seqNo)
285− (yieldOutput seqNo)
286−
287−-------------------------------------------------------------------------------
288−-- WAhead
289−-------------------------------------------------------------------------------
290−
291−-- XXX To be implemented. Use a linked queue like WAsync and put back the
292−-- remaining computation at the back of the queue instead of the heap, and
293−-- increment the sequence number.
294−
295−-- | Pull a stream from an SVar.
296−{-# NOINLINE fromStreamVar #-}
297−fromStreamVar :: MonadAsync m => SVar Stream m a -> Stream m a
298−fromStreamVar sv = Stream $ \_ stp sng yld -> do
299− list <- readOutputQ sv
300− -- Reversing the output is important to guarantee that we process the
301− -- outputs in the same order as they were generated by the constituent
302− -- streams.
303− runStream (processEvents $ reverse list) Nothing stp sng yld
304−
305− where
306−
307− allDone stp = do
308−#ifdef DIAGNOSTICS
309−#ifdef DIAGNOSTICS_VERBOSE
310− svInfo <- liftIO $ dumpSVar sv
311− liftIO $ hPutStrLn stderr $ "fromStreamVar done\n" ++ svInfo
312−#endif
313−#endif
314− stp
315−
316− {-# INLINE processEvents #-}
317− processEvents [] = Stream $ \_ stp sng yld -> do
318− done <- postProcess sv
319− if done
320− then allDone stp
321− else runStream (fromStreamVar sv) Nothing stp sng yld
322−
323− processEvents (ev : es) = Stream $ \_ stp sng yld -> do
324− let rest = processEvents es
325− case ev of
326− ChildYield a -> yld a rest
327− ChildStop tid e -> do
328− accountThread sv tid
329− case e of
330− Nothing -> runStream rest Nothing stp sng yld
331− Just ex -> throwM ex
332−
333−------------------------------------------------------------------------------
334−-- Running streams concurrently
335−------------------------------------------------------------------------------
336−
337−-- Concurrency rate control.
338−--
339−-- Our objective is to create more threads on demand if the consumer is running
340−-- faster than us. As soon as we encounter a concurrent composition we create a
341−-- push pull pair of threads. We use an SVar for communication between the
342−-- consumer, pulling from the SVar and the producer who is pushing to the SVar.
343−-- The producer creates more threads if the SVar drains and becomes empty, that
344−-- is the consumer is running faster.
345−--
346−-- XXX Note 1: This mechanism can be problematic if the initial production
347−-- latency is high, we may end up creating too many threads. So we need some
348−-- way to monitor and use the latency as well. Having a limit on the dispatches
349−-- (programmer controlled) may also help.
350−--
351−-- TBD Note 2: We may want to run computations at the lower level of the
352−-- composition tree serially even when they are composed using a parallel
353−-- combinator. We can use 'serial' in place of 'async' and 'wSerial' in
354−-- place of 'wAsync'. If we find that an SVar immediately above a computation
355−-- gets drained empty we can switch to parallelizing the computation. For that
356−-- we can use a state flag to fork the rest of the computation at any point of
357−-- time inside the Monad bind operation if the consumer is running at a faster
358−-- speed.
359−--
360−-- TBD Note 3: the binary operation ('parallel') composition allows us to
361−-- dispatch a chunkSize of only 1. If we have to dispatch in arbitrary
362−-- chunksizes we will need to compose the parallel actions using a data
363−-- constructor (A Free container) instead so that we can divide it in chunks of
364−-- arbitrary size before dispatching. If the stream is composed of
365−-- hierarchically composed grains of different sizes then we can always switch
366−-- to a desired granularity depending on the consumer speed.
367−--
368−-- TBD Note 4: for pure work (when we are not in the IO monad) we can divide it
369−-- into just the number of CPUs.
370−
371−-- | Join two computations on the currently running 'SVar' queue for concurrent
372−-- execution. When we are using parallel composition, an SVar is passed around
373−-- as a state variable. We try to schedule a new parallel computation on the
374−-- SVar passed to us. The first time, when no SVar exists, a new SVar is
375−-- created. Subsequently, 'joinStreamVarAsync' may get called when a computation
376−-- already scheduled on the SVar is further evaluated. For example, when (a
377−-- `parallel` b) is evaluated it calls a 'joinStreamVarAsync' to put 'a' and 'b' on
378−-- the current scheduler queue.
379−--
380−-- The 'SVarStyle' required by the current composition context is passed as one
381−-- of the parameters. If the scheduling and composition style of the new
382−-- computation being scheduled is different than the style of the current SVar,
383−-- then we create a new SVar and schedule it on that. The newly created SVar
384−-- joins as one of the computations on the current SVar queue.
385−--
386−-- Cases when we need to switch to a new SVar:
387−--
388−-- * (x `parallel` y) `parallel` (t `parallel` u) -- all of them get scheduled on the same SVar
389−-- * (x `parallel` y) `parallel` (t `async` u) -- @t@ and @u@ get scheduled on a new child SVar
390−-- because of the scheduling policy change.
391−-- * if we 'adapt' a stream of type 'async' to a stream of type
392−-- 'Parallel', we create a new SVar at the transitioning bind.
393−-- * When the stream is switching from disjunctive composition to conjunctive
394−-- composition and vice-versa we create a new SVar to isolate the scheduling
395−-- of the two.
396−
397−forkSVarAsync :: MonadAsync m => SVarStyle -> Stream m a -> Stream m a -> Stream m a
398−forkSVarAsync style m1 m2 = Stream $ \_ stp sng yld -> do
399− sv <- case style of
400− AsyncVar -> newAsyncVar (concurrently m1 m2)
401− WAsyncVar -> newWAsyncVar (concurrently m1 m2)
402− _ -> error "illegal svar type"
403− runStream (fromStreamVar sv) Nothing stp sng yld
404− where
405− concurrently ma mb = Stream $ \svr stp sng yld -> do
406− liftIO $ enqueue (fromJust svr) mb
407− (runStream ma) svr stp sng yld
408−
409−{-# INLINE joinStreamVarAsync #-}
410−joinStreamVarAsync :: MonadAsync m
411− => SVarStyle -> Stream m a -> Stream m a -> Stream m a
412−joinStreamVarAsync style m1 m2 = Stream $ \svr stp sng yld ->
413− case svr of
414− Just sv | svarStyle sv == style ->
415− liftIO (enqueue sv m2) >> (runStream m1) svr stp sng yld
416− _ -> runStream (forkSVarAsync style m1 m2) Nothing stp sng yld
417−
418−-- The only difference between forkSVarAsync and this is that we run the left
419−-- computation without a shared SVar.
420−forkSVarAhead :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
421−forkSVarAhead m1 m2 = Stream $ \_ stp sng yld -> do
422− sv <- newAheadVar (concurrently m1 m2) workLoopAhead
423− (runStream (fromStreamVar sv)) Nothing stp sng yld
424− where
425− concurrently ma mb = Stream $ \svr stp sng yld -> do
426− liftIO $ enqueue (fromJust svr) mb
427− (runStream ma) Nothing stp sng yld
428−
429−{-# INLINE ahead #-}
430−ahead :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
431−ahead m1 m2 = Stream $ \svr stp sng yld -> do
432− case svr of
433− Just sv | svarStyle sv == AheadVar -> do
434− liftIO $ enqueue sv m2
435− -- Always run the left side on a new SVar to avoid complexity in
436− -- sequencing results. This means the left side cannot further
437− -- split into more ahead computations on the same SVar.
438− (runStream m1) Nothing stp sng yld
439− _ -> runStream (forkSVarAhead m1 m2) Nothing stp sng yld
440−
441−{-# NOINLINE forkSVarPar #-}
442−forkSVarPar :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
443−forkSVarPar m r = Stream $ \_ stp sng yld -> do
444− sv <- newParallelVar
445− pushWorkerPar sv (runOne sv m)
446− pushWorkerPar sv (runOne sv r)
447− (runStream (fromStreamVar sv)) Nothing stp sng yld
448−
449−{-# INLINE joinStreamVarPar #-}
450−joinStreamVarPar :: MonadAsync m
451− => SVarStyle -> Stream m a -> Stream m a -> Stream m a
452−joinStreamVarPar style m1 m2 = Stream $ \svr stp sng yld ->
453− case svr of
454− Just sv | svarStyle sv == style -> do
455− pushWorkerPar sv (runOne sv m1) >> (runStream m2) svr stp sng yld
456− _ -> runStream (forkSVarPar m1 m2) Nothing stp sng yld
457−
458−------------------------------------------------------------------------------
459−-- Semigroup and Monoid style compositions for parallel actions
460−------------------------------------------------------------------------------
461−
462−{-# INLINE async #-}
463−async :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
464−async = joinStreamVarAsync AsyncVar
465−
466−-- | XXX we can implement it more efficienty by directly implementing instead
467−-- of combining streams using async.
468−{-# INLINE consMAsync #-}
469−consMAsync :: MonadAsync m => m a -> Stream m a -> Stream m a
470−consMAsync m r = once m `async` r
471−
472−{-# INLINE wAsync #-}
473−wAsync :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
474−wAsync = joinStreamVarAsync WAsyncVar
475−
476−-- | XXX we can implement it more efficienty by directly implementing instead
477−-- of combining streams using wAsync.
478−{-# INLINE consMWAsync #-}
479−consMWAsync :: MonadAsync m => m a -> Stream m a -> Stream m a
480−consMWAsync m r = once m `wAsync` r
481−
482−-- | XXX we can implement it more efficienty by directly implementing instead
483−-- of combining streams using ahead.
484−{-# INLINE consMAhead #-}
485−consMAhead :: MonadAsync m => m a -> Stream m a -> Stream m a
486−consMAhead m r = once m `ahead` r
487−
488−{-# INLINE parallel #-}
489−parallel :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
490−parallel = joinStreamVarPar ParallelVar
491−
492−-- | XXX we can implement it more efficienty by directly implementing instead
493−-- of combining streams using parallel.
494−{-# INLINE consMParallel #-}
495−consMParallel :: MonadAsync m => m a -> Stream m a -> Stream m a
496−consMParallel m r = once m `parallel` r
497−
498−------------------------------------------------------------------------------
499−-- Stream to stream function application
500−------------------------------------------------------------------------------
501−
502−mkParallel :: MonadAsync m => Stream m a -> m (Stream m a)
503−mkParallel m = do
504− sv <- newParallelVar
505− pushWorkerPar sv (runOne sv m)
506− return $ fromStreamVar sv
507−
508−applyWith :: MonadAsync m
509− => (Stream m a -> Stream m b) -> Stream m a -> Stream m b
510−applyWith f m = Stream $ \svr stp sng yld -> do
511− sv <- newParallelVar
512− pushWorkerPar sv (runOne sv m)
513− runStream (f $ fromStreamVar sv) svr stp sng yld
514−
515−------------------------------------------------------------------------------
516−-- Stream runner function application
517−------------------------------------------------------------------------------
518−
519−runWith :: MonadAsync m => (Stream m a -> m b) -> Stream m a -> m b
520−runWith f m = do
521− sv <- newParallelVar
522− pushWorkerPar sv (runOne sv m)
523− f $ fromStreamVar sv
524−
525−{-# INLINE zipAsyncWith #-}
526−zipAsyncWith :: MonadAsync m
527− => (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c
528−zipAsyncWith f m1 m2 = Stream $ \_ stp sng yld -> do
529− ma <- mkAsync m1
530− mb <- mkAsync m2
531− (runStream (zipWith f ma mb)) Nothing stp sng yld
532−
533− where
534−
535− mkAsync :: MonadAsync m => Stream m a -> m (Stream m a)
536− mkAsync m = newAsyncVar m
537− >>= return . fromStreamVar
538−
539−-------------------------------------------------------------------------------
540−-- SVar creation
541−-- This code belongs in SVar.hs but is kept here for perf reasons
542−-------------------------------------------------------------------------------
543−
544−-- XXX we have this function in this file because passing runStreamLIFO as a
545−-- function argument to this function results in a perf degradation of more
546−-- than 10%. Need to investigate what the root cause is.
547−-- Interestingly, the same thing does not make any difference for Ahead.
548−getLifoSVar :: MonadAsync m => IO (SVar Stream m a)
549−getLifoSVar = do
550− outQ <- newIORef ([], 0)
551− outQMv <- newEmptyMVar
552− active <- newIORef 0
553− wfw <- newIORef False
554− running <- newIORef S.empty
555− q <- newIORef []
556−#ifdef DIAGNOSTICS
557− disp <- newIORef 0
558− maxWrk <- newIORef 0
559− maxOq <- newIORef 0
560− maxHs <- newIORef 0
561− maxWq <- newIORef 0
562−#endif
563− let checkEmpty = null <$> readIORef q
564− let sv =
565− SVar { outputQueue = outQ
566− , outputDoorBell = outQMv
567− , readOutputQ = readOutputQBounded sv
568− , postProcess = postProcessBounded sv
569− , workerThreads = running
570− , workLoop = workLoopLIFO runStreamLIFO sv q
571− , enqueue = enqueueLIFO sv q
572− , isWorkDone = checkEmpty
573− , needDoorBell = wfw
574− , svarStyle = AsyncVar
575− , workerCount = active
576− , accountThread = delThread sv
577−#ifdef DIAGNOSTICS
578− , aheadWorkQueue = undefined
579− , outputHeap = undefined
580− , maxWorkers = maxWrk
581− , totalDispatches = disp
582− , maxOutQSize = maxOq
583− , maxHeapSize = maxHs
584− , maxWorkQSize = maxWq
585−#endif
586− }
587− in return sv
588−
589−getFifoSVar :: MonadAsync m => IO (SVar Stream m a)
590−getFifoSVar = do
591− outQ <- newIORef ([], 0)
592− outQMv <- newEmptyMVar
593− active <- newIORef 0
594− wfw <- newIORef False
595− running <- newIORef S.empty
596− q <- newQ
597−#ifdef DIAGNOSTICS
598− disp <- newIORef 0
599− maxWrk <- newIORef 0
600− maxOq <- newIORef 0
601− maxHs <- newIORef 0
602− maxWq <- newIORef 0
603−#endif
604− let sv =
605− SVar { outputQueue = outQ
606− , outputDoorBell = outQMv
607− , readOutputQ = readOutputQBounded sv
608− , postProcess = postProcessBounded sv
609− , workerThreads = running
610− , workLoop = workLoopFIFO runStreamFIFO sv q
611− , enqueue = enqueueFIFO sv q
612− , isWorkDone = nullQ q
613− , needDoorBell = wfw
614− , svarStyle = WAsyncVar
615− , workerCount = active
616− , accountThread = delThread sv
617−#ifdef DIAGNOSTICS
618− , aheadWorkQueue = undefined
619− , outputHeap = undefined
620− , totalDispatches = disp
621− , maxWorkers = maxWrk
622− , maxOutQSize = maxOq
623− , maxHeapSize = maxHs
624− , maxWorkQSize = maxWq
625−#endif
626− }
627− in return sv
628−
629−{-# INLINABLE newAsyncVar #-}
630−newAsyncVar :: MonadAsync m => Stream m a -> m (SVar Stream m a)
631−newAsyncVar m = do
632− sv <- liftIO getLifoSVar
633− sendWorker sv m
634−
635−-- | Create a new SVar and enqueue one stream computation on it.
636−{-# INLINABLE newWAsyncVar #-}
637−newWAsyncVar :: MonadAsync m => Stream m a -> m (SVar Stream m a)
638−newWAsyncVar m = do
639− sv <- liftIO getFifoSVar
640− sendWorker sv m
src/Streamly/CPS/Stream.hsdeleted−183
@@ -1,183 +0,0 @@
1−{-# LANGUAGE CPP #-}
2−{-# LANGUAGE ConstraintKinds #-}
3−{-# LANGUAGE FlexibleContexts #-}
4−{-# LANGUAGE FlexibleInstances #-}
5−{-# LANGUAGE LambdaCase #-}
6−{-# LANGUAGE MagicHash #-}
7−{-# LANGUAGE MultiParamTypeClasses #-}
8−{-# LANGUAGE RankNTypes #-}
9−{-# LANGUAGE UnboxedTuples #-}
10−{-# LANGUAGE UndecidableInstances #-} -- XXX
11−
12−-- |
13−-- Module : Streamly.CPS.Stream
14−-- Copyright : (c) 2017 Harendra Kumar
15−--
16−-- License : BSD3
17−-- Maintainer : harendra.kumar@gmail.com
18−-- Stability : experimental
19−-- Portability : GHC
20−--
21−--
22−module Streamly.CPS.Stream
23− (
24− -- * Streams
25− Stream (..)
26−
27− -- * Construction (pure)
28− , nil
29− , cons
30− , singleton
31− , once
32− , repeat
33−
34− -- * Construction (monadic)
35− , consM
36−
37− -- * Semigroup Style Composition
38− , serial
39− , wSerial
40−
41− -- * zip
42− , zipWith
43− )
44−where
45−
46−import Control.Monad.Trans.Class (MonadTrans(lift))
47−import Data.Semigroup (Semigroup(..))
48−import Prelude hiding (repeat, zipWith)
49−
50−import Streamly.SVar
51−
52−------------------------------------------------------------------------------
53−-- The stream type
54−------------------------------------------------------------------------------
55−
56−-- | The type 'Stream m a' represents a monadic stream of values of type 'a'
57−-- constructed using actions in monad 'm'. It uses stop, singleton and yield
58−-- continuations equivalent to the following direct style type:
59−--
60−-- data Stream m a = Stop | Singleton a | Yield a (Stream m a)
61−--
62−-- To facilitate parallel composition we maintain a local state in an SVar that
63−-- is shared across and is used for synchronization of the streams being
64−-- composed.
65−--
66−-- The singleton case can be expressed in terms of stop and yield but we have
67−-- it as a separate case to optimize composition operations for streams with
68−-- single element. We build singleton streams in the implementation of 'pure'
69−-- for Applicative and Monad, and in 'lift' for MonadTrans.
70−--
71−newtype Stream m a =
72− Stream {
73− runStream :: forall r.
74− Maybe (SVar Stream m a) -- local state
75− -> m r -- stop
76− -> (a -> m r) -- singleton
77− -> (a -> Stream m a -> m r) -- yield
78− -> m r
79− }
80−
81−nil :: Stream m a
82−nil = Stream $ \_ stp _ _ -> stp
83−
84−-- | faster than consM because there is no bind.
85−cons :: a -> Stream m a -> Stream m a
86−cons a r = Stream $ \_ _ _ yld -> yld a r
87−
88−-- | Same as @once . return@ but may be faster because there is no bind
89−singleton :: a -> Stream m a
90−singleton a = Stream $ \_ _ single _ -> single a
91−
92−{-# INLINE once #-}
93−once :: Monad m => m a -> Stream m a
94−once m = Stream $ \_ _ single _ -> m >>= single
95−
96−{-# INLINE consM #-}
97−consM :: Monad m => m a -> Stream m a -> Stream m a
98−consM m r = Stream $ \_ _ _ yld -> m >>= \a -> yld a r
99−
100−repeat :: a -> Stream m a
101−repeat a = let x = cons a x in x
102−
103−------------------------------------------------------------------------------
104−-- Semigroup
105−------------------------------------------------------------------------------
106−
107−-- | Concatenates two streams sequentially i.e. the first stream is
108−-- exhausted completely before yielding any element from the second stream.
109−{-# INLINE serial #-}
110−serial :: Stream m a -> Stream m a -> Stream m a
111−serial m1 m2 = go m1
112− where
113− go (Stream m) = Stream $ \_ stp sng yld ->
114− let stop = (runStream m2) Nothing stp sng yld
115− single a = yld a m2
116− yield a r = yld a (go r)
117− in m Nothing stop single yield
118−
119−instance Semigroup (Stream m a) where
120− (<>) = serial
121−
122−------------------------------------------------------------------------------
123−-- Monoid
124−------------------------------------------------------------------------------
125−
126−instance Monoid (Stream m a) where
127− mempty = nil
128− mappend = (<>)
129−
130−------------------------------------------------------------------------------
131−-- Interleave
132−------------------------------------------------------------------------------
133−
134−{-# INLINE wSerial #-}
135−wSerial :: Stream m a -> Stream m a -> Stream m a
136−wSerial m1 m2 = Stream $ \_ stp sng yld -> do
137− let stop = (runStream m2) Nothing stp sng yld
138− single a = yld a m2
139− yield a r = yld a (wSerial m2 r)
140− (runStream m1) Nothing stop single yield
141−
142−-------------------------------------------------------------------------------
143−-- Functor
144−-------------------------------------------------------------------------------
145−
146−instance Monad m => Functor (Stream m) where
147− fmap f m = Stream $ \_ stp sng yld ->
148− let single = sng . f
149− yield a r = yld (f a) (fmap f r)
150− in (runStream m) Nothing stp single yield
151−
152−------------------------------------------------------------------------------
153−-- Alternative & MonadPlus
154−------------------------------------------------------------------------------
155−
156−_alt :: Stream m a -> Stream m a -> Stream m a
157−_alt m1 m2 = Stream $ \_ stp sng yld ->
158− let stop = runStream m2 Nothing stp sng yld
159− in runStream m1 Nothing stop sng yld
160−
161−------------------------------------------------------------------------------
162−-- Zipping
163−------------------------------------------------------------------------------
164−
165−{-# INLINE zipWith #-}
166−zipWith :: (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c
167−zipWith f m1 m2 = go m1 m2
168− where
169− go mx my = Stream $ \_ stp sng yld -> do
170− let merge a ra =
171− let single2 b = sng (f a b)
172− yield2 b rb = yld (f a b) (go ra rb)
173− in (runStream my) Nothing stp single2 yield2
174− let single1 a = merge a nil
175− yield1 a ra = merge a ra
176− (runStream mx) Nothing stp single1 yield1
177−
178−-------------------------------------------------------------------------------
179−-- Transformers
180−-------------------------------------------------------------------------------
181−
182−instance MonadTrans Stream where
183− lift = once
src/Streamly/Prelude.hs+9−72
@@ -130,7 +130,6 @@ where
130130
131131import Control.Monad (void)
132132import Control.Monad.IO.Class (MonadIO(..))
133−import Data.Semigroup (Semigroup(..))
134133import Data.Maybe (isJust, fromJust)
135134import Prelude hiding (filter, drop, dropWhile, take,
136135 takeWhile, zipWith, foldr, foldl,
@@ -142,9 +141,11 @@ import Prelude hiding (filter, drop, dropWhile, take,
142141import qualified Prelude
143142import qualified System.IO as IO
144143
145−import qualified Streamly.CPS.Stream as C
146−import qualified Streamly.CPS.Concurrent as CC
147−import Streamly.Streams
144+import Streamly.Streams.CPS
145+import qualified Streamly.Streams.CPS as C
146+import Streamly.Streams.Zip
147+import Streamly.Streams.Serial
148+import Streamly.SVar (MonadAsync)
148149
149150------------------------------------------------------------------------------
150151-- Construction
@@ -247,19 +248,6 @@ fromFoldableM = Prelude.foldr consM nil
247248each :: (IsStream t, Foldable f) => f a -> t m a
248249each = fromFoldable
249250
250−-- | Create a singleton stream by executing a monadic action once. Same as
251−-- @m \`consM` nil@ but more efficient.
252−--
253−-- @
254−-- > toList $ once getLine
255−-- hello
256−-- ["hello"]
257−-- @
258−--
259−-- @since 0.2.0
260−once :: (IsStream t, Monad m) => m a -> t m a
261−once = fromStream . C.once
262−
263251-- | Generate a stream by performing a monadic action @n@ times.
264252--
265253--
@@ -693,10 +681,10 @@ reverse :: (IsStream t) => t m a -> t m a
693681reverse m = fromStream $ go C.nil (toStream m)
694682 where
695683 go rev rest = C.Stream $ \_ stp sng yld ->
696− let run x = C.runStream x Nothing stp sng yld
697− stop = run rev
698− single a = run $ a `C.cons` rev
699− yield a r = run $ go (a `C.cons` rev) r
684+ let runIt x = C.runStream x Nothing stp sng yld
685+ stop = runIt rev
686+ single a = runIt $ a `C.cons` rev
687+ yield a r = runIt $ go (a `C.cons` rev) r
700688 in C.runStream rest Nothing stop single yield
701689
702690-- XXX replace the recursive "go" with continuation
@@ -822,54 +810,3 @@ sequence m = go (toStream m)
822810 let single ma = ma >>= sng
823811 yield ma r = C.runStream (toStream $ ma |: go r) svr stp sng yld
824812 in (C.runStream m1) Nothing stp single yield
825−
826−------------------------------------------------------------------------------
827−-- Serially Zipping Streams
828−------------------------------------------------------------------------------
829−
830−-- | Zip two streams serially using a pure zipping function.
831−--
832−-- @since 0.1.0
833−{-# INLINABLE zipWith #-}
834−zipWith :: IsStream t => (a -> b -> c) -> t m a -> t m b -> t m c
835−zipWith f m1 m2 = fromStream $ C.zipWith f (toStream m1) (toStream m2)
836−
837−-- | Zip two streams serially using a monadic zipping function.
838−--
839−-- @since 0.1.0
840−zipWithM :: IsStream t => (a -> b -> t m c) -> t m a -> t m b -> t m c
841−zipWithM f m1 m2 = fromStream $ go (toStream m1) (toStream m2)
842− where
843− go mx my = C.Stream $ \_ stp sng yld -> do
844− let merge a ra =
845− let run x = C.runStream x Nothing stp sng yld
846− single2 b = run $ toStream (f a b)
847− yield2 b rb = run $ toStream (f a b) <> go ra rb
848− in (C.runStream my) Nothing stp single2 yield2
849− let single1 a = merge a C.nil
850− yield1 a ra = merge a ra
851− (C.runStream mx) Nothing stp single1 yield1
852−
853−------------------------------------------------------------------------------
854−-- Parallely Zipping Streams
855−------------------------------------------------------------------------------
856−
857−-- | Zip two streams concurrently (i.e. both the elements being zipped are
858−-- generated concurrently) using a pure zipping function.
859−--
860−-- @since 0.1.0
861−zipAsyncWith :: (IsStream t, MonadAsync m)
862− => (a -> b -> c) -> t m a -> t m b -> t m c
863−zipAsyncWith f m1 m2 =
864− fromStream $ CC.zipAsyncWith f (toStream m1) (toStream m2)
865−
866−-- | Zip two streams asyncly (i.e. both the elements being zipped are generated
867−-- concurrently) using a monadic zipping function.
868−--
869−-- @since 0.1.0
870−zipAsyncWithM :: (IsStream t, MonadAsync m)
871− => (a -> b -> t m c) -> t m a -> t m b -> t m c
872−zipAsyncWithM f m1 m2 = fromStream $ C.Stream $ \_ stp sng yld -> do
873− ma <- mkAsync m1
874− mb <- mkAsync m2
875− (C.runStream (toStream (zipWithM f ma mb))) Nothing stp sng yld
src/Streamly/Streams.hsdeleted−1489
@@ -1,1489 +0,0 @@
1−{-# LANGUAGE CPP #-}
2−{-# LANGUAGE ConstraintKinds #-}
3−{-# LANGUAGE FlexibleContexts #-}
4−{-# LANGUAGE FlexibleInstances #-}
5−{-# LANGUAGE GeneralizedNewtypeDeriving#-}
6−{-# LANGUAGE InstanceSigs #-}
7−{-# LANGUAGE MultiParamTypeClasses #-}
8−{-# LANGUAGE RankNTypes #-}
9−{-# LANGUAGE StandaloneDeriving #-}
10−{-# LANGUAGE UndecidableInstances #-} -- XXX
11−
12−-- |
13−-- Module : Streamly.Streams
14−-- Copyright : (c) 2017 Harendra Kumar
15−--
16−-- License : BSD3
17−-- Maintainer : harendra.kumar@gmail.com
18−-- Stability : experimental
19−-- Portability : GHC
20−--
21−--
22−module Streamly.Streams
23− (
24− IsStream (..)
25− , Streaming -- deprecated
26− , MonadAsync
27−
28− -- * Construction
29− , nil
30− , cons
31− , (.:)
32− , streamBuild
33− , fromCallback
34− , fromSVar
35−
36− -- * Elimination
37− , streamFold
38− , runStream
39− , runStreaming -- deprecated
40− , toSVar
41−
42− -- * Transformation
43− , mkAsync
44− , (|$)
45− , (|&)
46− , (|$.)
47− , (|&.)
48−
49− -- * Merging Streams
50− , serial
51− , wSerial
52− , ahead
53− , async
54− , wAsync
55− , parallel
56− , (<=>) --deprecated
57− , (<|) --deprecated
58−
59− -- * IO Streams
60− , Serial
61− , WSerial
62− , Ahead
63− , Async
64− , WAsync
65− , Parallel
66− , ZipSerial
67− , ZipAsync
68−
69− -- * Stream Transformers
70− , SerialT
71− , StreamT -- deprecated
72− , WSerialT
73− , InterleavedT -- deprecated
74− , AheadT
75− , AsyncT
76− , WAsyncT
77− , ParallelT
78− , ZipStream -- deprecated
79− , ZipSerialM
80− , ZipAsyncM
81−
82− -- * Type Adapters
83− , serially -- deprecated
84− , wSerially
85− , interleaving -- deprecated
86− , aheadly
87− , asyncly
88− , wAsyncly
89− , parallely
90− , zipSerially
91− , zipping -- deprecated
92− , zipAsyncly
93− , zippingAsync -- deprecated
94− , adapt
95−
96− -- * Running Streams
97− , runStreamT -- deprecated
98− , runInterleavedT -- deprecated
99− , runAsyncT -- deprecated
100− , runParallelT -- deprecated
101− , runZipStream -- deprecated
102− , runZipAsync -- deprecated
103−
104− -- * Fold Utilities
105− , foldWith
106− , foldMapWith
107− , forEachWith
108− )
109−where
110−
111−import Control.Monad (ap)
112−import Control.Monad.Base (MonadBase (..), liftBaseDefault)
113−import Control.Monad.Catch (MonadThrow, throwM)
114−-- import Control.Monad.Error.Class (MonadError(..))
115−import Control.Monad.IO.Class (MonadIO(..))
116−import Control.Monad.Reader.Class (MonadReader(..))
117−import Control.Monad.State.Class (MonadState(..))
118−import Control.Monad.Trans.Class (MonadTrans (lift))
119−import Data.Semigroup (Semigroup(..))
120−import Streamly.SVar (MonadAsync, SVar, toStreamVar)
121−
122−import qualified Streamly.CPS.Stream as C
123−import qualified Streamly.CPS.Concurrent as CC
124−
125−------------------------------------------------------------------------------
126−-- Types that can behave as a Stream
127−------------------------------------------------------------------------------
128−
129−infixr 5 `consM`
130−infixr 5 |:
131−
132−-- | Class of types that can represent a stream of elements of some type 'a' in
133−-- some monad 'm'.
134−--
135−-- @since 0.2.0
136−class IsStream t where
137− toStream :: t m a -> C.Stream m a
138− fromStream :: C.Stream m a -> t m a
139− -- | Constructs a stream by adding a monadic action at the head of an
140− -- existing stream. For example:
141− --
142− -- @
143− -- > toList $ getLine \`consM` getLine \`consM` nil
144− -- hello
145− -- world
146− -- ["hello","world"]
147− -- @
148− --
149− -- /Concurrent (do not use 'parallely' to construct infinite streams)/
150− --
151− -- @since 0.2.0
152− consM :: MonadAsync m => m a -> t m a -> t m a
153− -- | Operator equivalent of 'consM'. We can read it as "@parallel colon@"
154− -- to remember that @|@ comes before ':'.
155− --
156− -- @
157− -- > toList $ getLine |: getLine |: nil
158− -- hello
159− -- world
160− -- ["hello","world"]
161− -- @
162− --
163− -- @
164− -- let delay = threadDelay 1000000 >> print 1
165− -- runStream $ serially $ delay |: delay |: delay |: nil
166− -- runStream $ parallely $ delay |: delay |: delay |: nil
167− -- @
168− --
169− -- /Concurrent (do not use 'parallely' to construct infinite streams)/
170− --
171− -- @since 0.2.0
172− (|:) :: MonadAsync m => m a -> t m a -> t m a
173− -- We can define (|:) just as 'consM' but it is defined explicitly for each
174− -- type because we want to use SPECIALIZE pragma on the definition.
175−
176−-- | Same as 'IsStream'.
177−--
178−-- @since 0.1.0
179−{-# DEPRECATED Streaming "Please use IsStream instead." #-}
180−type Streaming = IsStream
181−
182−------------------------------------------------------------------------------
183−-- Constructing a stream
184−------------------------------------------------------------------------------
185−
186−-- | An empty stream.
187−--
188−-- @
189−-- > toList nil
190−-- []
191−-- @
192−--
193−-- @since 0.1.0
194−nil :: IsStream t => t m a
195−nil = fromStream C.nil
196−
197−-- | Constructs a stream by adding a monadic action at the head of an existing
198−-- stream. For example:
199−--
200−-- @
201−-- > toList $ getLine \`consM` getLine \`consM` nil
202−-- hello
203−-- world
204−-- ["hello","world"]
205−-- @
206−--
207−-- @since 0.2.0
208−consMSerial :: (IsStream t, Monad m) => m a -> t m a -> t m a
209−consMSerial m r = fromStream $ C.consM m (toStream r)
210−
211−infixr 5 `cons`
212−
213−-- | Construct a stream by adding a pure value at the head of an existing
214−-- stream. For pure values it can be faster than 'consM'. For example:
215−--
216−-- @
217−-- > toList $ 1 \`cons` 2 \`cons` 3 \`cons` nil
218−-- [1,2,3]
219−-- @
220−--
221−-- @since 0.1.0
222−cons :: IsStream t => a -> t m a -> t m a
223−cons a r = fromStream $ C.cons a (toStream r)
224−
225−infixr 5 .:
226−
227−-- | Operator equivalent of 'cons'.
228−--
229−-- @
230−-- > toList $ 1 .: 2 .: 3 .: nil
231−-- [1,2,3]
232−-- @
233−--
234−-- @since 0.1.1
235−(.:) :: IsStream t => a -> t m a -> t m a
236−(.:) = cons
237−
238−-- | Build a stream from its church encoding. The function passed maps
239−-- directly to the underlying representation of the stream type. The second
240−-- parameter to the function is the "yield" function yielding a value and the
241−-- remaining stream if any otherwise 'Nothing'. The third parameter is to
242−-- represent an "empty" stream.
243−streamBuild :: IsStream t
244− => (forall r. Maybe (SVar C.Stream m a)
245− -> (a -> t m a -> m r)
246− -> (a -> m r)
247− -> m r
248− -> m r)
249− -> t m a
250−streamBuild k = fromStream $ C.Stream $ \svr stp sng yld ->
251− let yield a r = yld a (toStream r)
252− in k svr yield sng stp
253−
254−-- | Build a singleton stream from a callback function.
255−fromCallback :: IsStream t => (forall r. (a -> m r) -> m r) -> t m a
256−fromCallback k = fromStream $ C.Stream $ \_ _ sng _ -> k sng
257−
258−-- | Read an SVar to get a stream.
259−fromSVar :: (MonadAsync m, IsStream t) => SVar C.Stream m a -> t m a
260−fromSVar sv = fromStream $ CC.fromStreamVar sv
261−
262−------------------------------------------------------------------------------
263−-- Destroying a stream
264−------------------------------------------------------------------------------
265−
266−-- | Fold a stream using its church encoding. The second argument is the "step"
267−-- function consuming an element and the remaining stream, if any. The third
268−-- argument is for consuming an "empty" stream that yields nothing.
269−streamFold
270− :: IsStream t
271− => Maybe (SVar C.Stream m a)
272− -> (a -> t m a -> m r)
273− -> (a -> m r)
274− -> m r
275− -> t m a
276− -> m r
277−streamFold svr step single blank m =
278− let yield a x = step a (fromStream x)
279− in (C.runStream (toStream m)) svr blank single yield
280−
281−-- | Run a streaming composition, discard the results. By default it interprets
282−-- the stream as 'SerialT', to run other types of streams use the type adapting
283−-- combinators for example @runStream . 'asyncly'@.
284−--
285−-- @since 0.2.0
286−runStream :: Monad m => SerialT m a -> m ()
287−runStream m = go (toStream m)
288− where
289− go m1 =
290− let stop = return ()
291− single _ = return ()
292− yield _ r = go r
293− in (C.runStream m1) Nothing stop single yield
294−
295−-- | Same as 'runStream'
296−--
297−-- @since 0.1.0
298−{-# DEPRECATED runStreaming "Please use runStream instead." #-}
299−runStreaming :: (Monad m, IsStream t) => t m a -> m ()
300−runStreaming = runStream . adapt
301−
302−-- | Write a stream to an 'SVar' in a non-blocking manner. The stream can then
303−-- be read back from the SVar using 'fromSVar'.
304−toSVar :: (IsStream t, MonadAsync m) => SVar C.Stream m a -> t m a -> m ()
305−toSVar sv m = toStreamVar sv (toStream m)
306−
307−------------------------------------------------------------------------------
308−-- Transformation
309−------------------------------------------------------------------------------
310−
311−-- XXX Get rid of this?
312−-- | Make a stream asynchronous, triggers the computation and returns a stream
313−-- in the underlying monad representing the output generated by the original
314−-- computation. The returned action is exhaustible and must be drained once. If
315−-- not drained fully we may have a thread blocked forever and once exhausted it
316−-- will always return 'empty'.
317−--
318−-- @since 0.2.0
319−mkAsync :: (IsStream t, MonadAsync m) => t m a -> m (t m a)
320−mkAsync m = do
321− s <- CC.mkParallel (toStream m)
322− return $ fromStream s
323−
324−{-# INLINE applyWith #-}
325−applyWith :: (IsStream t, MonadAsync m) => (t m a -> t m b) -> t m a -> t m b
326−applyWith f x = fromStream $
327− CC.applyWith (toStream . f . fromStream) (toStream x)
328−
329−{-# INLINE runWith #-}
330−runWith :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> m b
331−runWith f x = CC.runWith (f . fromStream) (toStream x)
332−
333−infixr 0 |$
334−infixr 0 |$.
335−
336−infixl 1 |&
337−infixl 1 |&.
338−
339−-- | Parallel function application operator for streams; just like the regular
340−-- function application operator '$' except that it is concurrent. The
341−-- following code prints a value every second even though each stage adds a 1
342−-- second delay.
343−--
344−--
345−-- @
346−-- runStream $
347−-- S.mapM (\\x -> threadDelay 1000000 >> print x)
348−-- |$ S.repeatM (threadDelay 1000000 >> return 1)
349−-- @
350−--
351−-- /Concurrent/
352−--
353−-- @since 0.3.0
354−{-# INLINE (|$) #-}
355−(|$) :: (IsStream t, MonadAsync m) => (t m a -> t m b) -> t m a -> t m b
356−f |$ x = applyWith f x
357−
358−-- | Parallel reverse function application operator for streams; just like the
359−-- regular reverse function application operator '&' except that it is
360−-- concurrent.
361−--
362−-- @
363−-- runStream $
364−-- S.repeatM (threadDelay 1000000 >> return 1)
365−-- |& S.mapM (\\x -> threadDelay 1000000 >> print x)
366−-- @
367−--
368−-- /Concurrent/
369−--
370−-- @since 0.3.0
371−{-# INLINE (|&) #-}
372−(|&) :: (IsStream t, MonadAsync m) => t m a -> (t m a -> t m b) -> t m b
373−x |& f = f |$ x
374−
375−-- | Parallel function application operator; applies a @run@ or @fold@ function
376−-- to a stream such that the fold consumer and the stream producer run in
377−-- parallel. A @run@ or @fold@ function reduces the stream to a value in the
378−-- underlying monad. The @.@ at the end of the operator is a mnemonic for
379−-- termination of the stream.
380−--
381−-- @
382−-- S.foldlM' (\\_ a -> threadDelay 1000000 >> print a) ()
383−-- |$. S.repeatM (threadDelay 1000000 >> return 1)
384−-- @
385−--
386−-- /Concurrent/
387−--
388−-- @since 0.3.0
389−{-# INLINE (|$.) #-}
390−(|$.) :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> m b
391−f |$. x = runWith f x
392−
393−-- | Parallel reverse function application operator for applying a run or fold
394−-- functions to a stream. Just like '|$.' except that the operands are reversed.
395−--
396−-- @
397−-- S.repeatM (threadDelay 1000000 >> return 1)
398−-- |&. S.foldlM' (\\_ a -> threadDelay 1000000 >> print a) ()
399−-- @
400−--
401−-- /Concurrent/
402−--
403−-- @since 0.3.0
404−{-# INLINE (|&.) #-}
405−(|&.) :: (IsStream t, MonadAsync m) => t m a -> (t m a -> m b) -> m b
406−x |&. f = f |$. x
407−
408−------------------------------------------------------------------------------
409−-- CPP macros for common instances
410−------------------------------------------------------------------------------
411−
412−-- XXX use template haskell instead and include Monoid and IsStream instances
413−-- as well.
414−
415−withLocal :: MonadReader r m => (r -> r) -> C.Stream m a -> C.Stream m a
416−withLocal f m =
417− C.Stream $ \_ stp sng yld ->
418− let single = local f . sng
419− yield a r = local f $ yld a (withLocal f r)
420− in (C.runStream m) Nothing (local f stp) single yield
421−
422−{-
423−-- XXX handle and test cross thread state transfer
424−withCatchError
425− :: MonadError e m
426− => C.Stream m a -> (e -> C.Stream m a) -> C.Stream m a
427−withCatchError m h =
428− C.Stream $ \_ stp sng yld ->
429− let run x = C.runStream x Nothing stp sng yield
430− handle r = r `catchError` \e -> run $ h e
431− yield a r = yld a (withCatchError r h)
432− in handle $ run m
433−-}
434−
435−#define MONADPARALLEL , MonadAsync m
436−
437−#define MONAD_APPLICATIVE_INSTANCE(STREAM,CONSTRAINT) \
438−instance (Monad m CONSTRAINT) => Applicative (STREAM m) where { \
439− pure = STREAM . C.singleton; \
440− (<*>) = ap }
441−
442−#define MONAD_COMMON_INSTANCES(STREAM,CONSTRAINT) \
443−instance (MonadBase b m, Monad m CONSTRAINT) => MonadBase b (STREAM m) where {\
444− liftBase = liftBaseDefault }; \
445− \
446−instance (MonadIO m CONSTRAINT) => MonadIO (STREAM m) where { \
447− liftIO = lift . liftIO }; \
448− \
449−instance (MonadThrow m CONSTRAINT) => MonadThrow (STREAM m) where { \
450− throwM = lift . throwM }; \
451− \
452−{- \
453−instance (MonadError e m CONSTRAINT) => MonadError e (STREAM m) where { \
454− throwError = lift . throwError; \
455− catchError m h = \
456− fromStream $ withCatchError (toStream m) (\e -> toStream $ h e) }; \
457−-} \
458− \
459−instance (MonadReader r m CONSTRAINT) => MonadReader r (STREAM m) where { \
460− ask = lift ask; \
461− local f m = fromStream $ withLocal f (toStream m) }; \
462− \
463−instance (MonadState s m CONSTRAINT) => MonadState s (STREAM m) where { \
464− get = lift get; \
465− put x = lift (put x); \
466− state k = lift (state k) }
467−
468−------------------------------------------------------------------------------
469−-- SerialT
470−------------------------------------------------------------------------------
471−
472−-- | Deep serial composition or serial composition with depth first traversal.
473−-- The 'Semigroup' instance of 'SerialT' appends two streams serially in a
474−-- depth first manner, yielding all elements from the first stream, and then
475−-- all elements from the second stream.
476−--
477−-- @
478−-- import Streamly
479−-- import qualified "Streamly.Prelude" as S
480−--
481−-- main = ('toList' . 'serially' $ (fromFoldable [1,2]) \<\> (fromFoldable [3,4])) >>= print
482−-- @
483−-- @
484−-- [1,2,3,4]
485−-- @
486−--
487−-- The 'Monad' instance runs the /monadic continuation/ for each
488−-- element of the stream, serially.
489−--
490−-- @
491−-- main = 'runStream' . 'serially' $ do
492−-- x <- return 1 \<\> return 2
493−-- S.once $ print x
494−-- @
495−-- @
496−-- 1
497−-- 2
498−-- @
499−--
500−-- 'SerialT' nests streams serially in a depth first manner.
501−--
502−-- @
503−-- main = 'runStream' . 'serially' $ do
504−-- x <- return 1 \<\> return 2
505−-- y <- return 3 \<\> return 4
506−-- S.once $ print (x, y)
507−-- @
508−-- @
509−-- (1,3)
510−-- (1,4)
511−-- (2,3)
512−-- (2,4)
513−-- @
514−--
515−-- This behavior of 'SerialT' is exactly like a list transformer. We call the
516−-- monadic code being run for each element of the stream a monadic
517−-- continuation. In imperative paradigm we can think of this composition as
518−-- nested @for@ loops and the monadic continuation is the body of the loop. The
519−-- loop iterates for all elements of the stream.
520−--
521−-- The 'serially' combinator can be omitted as the default stream type is
522−-- 'SerialT'.
523−-- Note that serial composition with depth first traversal can be used to
524−-- combine an infinite number of streams as it explores only one stream at a
525−-- time.
526−--
527−-- @since 0.2.0
528−newtype SerialT m a = SerialT {getSerialT :: C.Stream m a}
529− deriving (Semigroup, Monoid, Functor, MonadTrans)
530−
531−-- |
532−-- @since 0.1.0
533−{-# DEPRECATED StreamT "Please use 'SerialT' instead." #-}
534−type StreamT = SerialT
535−
536−instance IsStream SerialT where
537− toStream = getSerialT
538− fromStream = SerialT
539−
540− {-# INLINE consM #-}
541− {-# SPECIALIZE consM :: IO a -> SerialT IO a -> SerialT IO a #-}
542− consM :: Monad m => m a -> SerialT m a -> SerialT m a
543− consM = consMSerial
544−
545− {-# INLINE (|:) #-}
546− {-# SPECIALIZE (|:) :: IO a -> SerialT IO a -> SerialT IO a #-}
547− (|:) :: Monad m => m a -> SerialT m a -> SerialT m a
548− (|:) = consMSerial
549−
550−------------------------------------------------------------------------------
551−-- Semigroup
552−------------------------------------------------------------------------------
553−
554−-- | Polymorphic version of the 'Semigroup' operation '<>' of 'SerialT'.
555−-- Appends two streams sequentially, yielding all elements from the first
556−-- stream, and then all elements from the second stream.
557−--
558−-- @since 0.2.0
559−{-# INLINE serial #-}
560−serial :: IsStream t => t m a -> t m a -> t m a
561−serial m1 m2 = fromStream $ C.serial (toStream m1) (toStream m2)
562−
563−------------------------------------------------------------------------------
564−-- Monad
565−------------------------------------------------------------------------------
566−
567−instance Monad m => Monad (SerialT m) where
568− return = pure
569− (SerialT (C.Stream m)) >>= f = SerialT $ C.Stream $ \_ stp sng yld ->
570− let run x = (C.runStream x) Nothing stp sng yld
571− single a = run $ toStream (f a)
572− yield a r = run $ toStream $ f a <> (fromStream r >>= f)
573− in m Nothing stp single yield
574−
575−------------------------------------------------------------------------------
576−-- Other instances
577−------------------------------------------------------------------------------
578−
579−MONAD_APPLICATIVE_INSTANCE(SerialT,)
580−MONAD_COMMON_INSTANCES(SerialT,)
581−
582−------------------------------------------------------------------------------
583−-- WSerialT
584−------------------------------------------------------------------------------
585−
586−-- | Wide serial composition or serial composition with a breadth first
587−-- traversal. The 'Semigroup' instance of 'WSerialT' traverses
588−-- the two streams in a breadth first manner. In other words, it interleaves
589−-- two streams, yielding one element from each stream alternately.
590−--
591−-- @
592−-- import Streamly
593−-- import qualified "Streamly.Prelude" as S
594−--
595−-- main = ('toList' . 'wSerially' $ (fromFoldable [1,2]) \<\> (fromFoldable [3,4])) >>= print
596−-- @
597−-- @
598−-- [1,3,2,4]
599−-- @
600−--
601−-- Similarly, the 'Monad' instance interleaves the iterations of the
602−-- inner and the outer loop, nesting loops in a breadth first manner.
603−--
604−--
605−-- @
606−-- main = 'runStream' . 'wSerially' $ do
607−-- x <- return 1 \<\> return 2
608−-- y <- return 3 \<\> return 4
609−-- S.once $ print (x, y)
610−-- @
611−-- @
612−-- (1,3)
613−-- (2,3)
614−-- (1,4)
615−-- (2,4)
616−-- @
617−--
618−-- Note that a serial composition with breadth first traversal can only combine
619−-- a finite number of streams as it needs to retain state for each unfinished
620−-- stream.
621−--
622−-- @since 0.2.0
623−newtype WSerialT m a = WSerialT {getWSerialT :: C.Stream m a}
624− deriving (Functor, MonadTrans)
625−
626−-- |
627−-- @since 0.1.0
628−{-# DEPRECATED InterleavedT "Please use 'WSerialT' instead." #-}
629−type InterleavedT = WSerialT
630−
631−instance IsStream WSerialT where
632− toStream = getWSerialT
633− fromStream = WSerialT
634−
635− {-# INLINE consM #-}
636− {-# SPECIALIZE consM :: IO a -> WSerialT IO a -> WSerialT IO a #-}
637− consM :: Monad m => m a -> WSerialT m a -> WSerialT m a
638− consM = consMSerial
639−
640− {-# INLINE (|:) #-}
641− {-# SPECIALIZE (|:) :: IO a -> WSerialT IO a -> WSerialT IO a #-}
642− (|:) :: Monad m => m a -> WSerialT m a -> WSerialT m a
643− (|:) = consMSerial
644−
645−------------------------------------------------------------------------------
646−-- Semigroup
647−------------------------------------------------------------------------------
648−
649−-- | Polymorphic version of the 'Semigroup' operation '<>' of 'WSerialT'.
650−-- Interleaves two streams, yielding one element from each stream alternately.
651−--
652−-- @since 0.2.0
653−{-# INLINE wSerial #-}
654−wSerial :: IsStream t => t m a -> t m a -> t m a
655−wSerial m1 m2 = fromStream $ C.wSerial (toStream m1) (toStream m2)
656−
657−instance Semigroup (WSerialT m a) where
658− (<>) = wSerial
659−
660−infixr 5 <=>
661−
662−-- | Same as 'wSerial'.
663−--
664−-- @since 0.1.0
665−{-# DEPRECATED (<=>) "Please use 'wSerial' instead." #-}
666−{-# INLINE (<=>) #-}
667−(<=>) :: IsStream t => t m a -> t m a -> t m a
668−(<=>) = wSerial
669−
670−------------------------------------------------------------------------------
671−-- Monoid
672−------------------------------------------------------------------------------
673−
674−instance Monoid (WSerialT m a) where
675− mempty = nil
676− mappend = (<>)
677−
678−------------------------------------------------------------------------------
679−-- Monad
680−------------------------------------------------------------------------------
681−
682−instance Monad m => Monad (WSerialT m) where
683− return = pure
684− (WSerialT (C.Stream m)) >>= f = WSerialT $ C.Stream $ \_ stp sng yld ->
685− let run x = (C.runStream x) Nothing stp sng yld
686− single a = run $ toStream (f a)
687− yield a r = run $ toStream $ f a <> (fromStream r >>= f)
688− in m Nothing stp single yield
689−
690−------------------------------------------------------------------------------
691−-- Other instances
692−------------------------------------------------------------------------------
693−
694−MONAD_APPLICATIVE_INSTANCE(WSerialT,)
695−MONAD_COMMON_INSTANCES(WSerialT,)
696−
697−------------------------------------------------------------------------------
698−-- AheadT
699−------------------------------------------------------------------------------
700−
701−-- | Deep ahead composition or ahead composition with depth first traversal.
702−-- The semigroup composition of 'AheadT' appends streams in a depth first
703−-- manner just like 'SerialT' except that it can produce elements concurrently
704−-- ahead of time. It is like 'AsyncT' except that 'AsyncT' produces the output
705−-- as it arrives whereas 'AheadT' orders the output in the traversal order.
706−--
707−-- @
708−-- main = ('toList' . 'aheadly' $ (fromFoldable [1,2]) \<> (fromFoldable [3,4])) >>= print
709−-- @
710−-- @
711−-- [1,2,3,4]
712−-- @
713−--
714−-- Any exceptions generated by a constituent stream are propagated to the
715−-- output stream.
716−--
717−-- Similarly, the monad instance of 'AheadT' may run each iteration
718−-- concurrently ahead of time but presents the results in the same order as
719−-- 'SerialT'.
720−--
721−-- @
722−-- import "Streamly"
723−-- import qualified "Streamly.Prelude" as S
724−-- import Control.Concurrent
725−--
726−-- main = 'runStream' . 'aheadly' $ do
727−-- n <- return 3 \<\> return 2 \<\> return 1
728−-- S.once $ do
729−-- threadDelay (n * 1000000)
730−-- myThreadId >>= \\tid -> putStrLn (show tid ++ ": Delay " ++ show n)
731−-- @
732−-- @
733−-- ThreadId 40: Delay 1
734−-- ThreadId 39: Delay 2
735−-- ThreadId 38: Delay 3
736−-- @
737−--
738−-- All iterations may run in the same thread if they do not block.
739−--
740−-- Note that ahead composition with depth first traversal can be used to
741−-- combine infinite number of streams as it explores only a bounded number of
742−-- streams at a time.
743−--
744−-- @since 0.3.0
745−newtype AheadT m a = AheadT {getAheadT :: C.Stream m a}
746− deriving (Functor, MonadTrans)
747−
748−instance IsStream AheadT where
749− toStream = getAheadT
750− fromStream = AheadT
751−
752− {-# INLINE consM #-}
753− {-# SPECIALIZE consM :: IO a -> AheadT IO a -> AheadT IO a #-}
754− consM m r = fromStream $ CC.consMAhead m (toStream r)
755−
756− {-# INLINE (|:) #-}
757− {-# SPECIALIZE (|:) :: IO a -> AheadT IO a -> AheadT IO a #-}
758− (|:) = consM
759−
760−------------------------------------------------------------------------------
761−-- Semigroup
762−------------------------------------------------------------------------------
763−
764−-- | Polymorphic version of the 'Semigroup' operation '<>' of 'AheadT'.
765−-- Merges two streams sequentially but with concurrent lookahead.
766−--
767−-- @since 0.3.0
768−{-# INLINE ahead #-}
769−ahead :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
770−ahead m1 m2 = fromStream $ CC.ahead (toStream m1) (toStream m2)
771−
772−instance MonadAsync m => Semigroup (AheadT m a) where
773− (<>) = ahead
774−
775−------------------------------------------------------------------------------
776−-- Monoid
777−------------------------------------------------------------------------------
778−
779−instance MonadAsync m => Monoid (AheadT m a) where
780− mempty = nil
781− mappend = (<>)
782−
783−------------------------------------------------------------------------------
784−-- Monad
785−------------------------------------------------------------------------------
786−
787−{-# INLINE aheadbind #-}
788−aheadbind
789− :: MonadAsync m
790− => C.Stream m a
791− -> (a -> C.Stream m b)
792− -> C.Stream m b
793−aheadbind m f = go m
794− where
795− go (C.Stream g) =
796− C.Stream $ \ctx stp sng yld ->
797− let run x = (C.runStream x) ctx stp sng yld
798− single a = run $ f a
799− yield a r = run $ f a `CC.ahead` go r
800− in g Nothing stp single yield
801−
802−instance MonadAsync m => Monad (AheadT m) where
803− return = pure
804− (AheadT m) >>= f = AheadT $ aheadbind m (getAheadT . f)
805−
806−------------------------------------------------------------------------------
807−-- Other instances
808−------------------------------------------------------------------------------
809−
810−MONAD_APPLICATIVE_INSTANCE(AheadT,MONADPARALLEL)
811−MONAD_COMMON_INSTANCES(AheadT, MONADPARALLEL)
812−
813−------------------------------------------------------------------------------
814−-- AsyncT
815−------------------------------------------------------------------------------
816−
817−-- | Deep async composition or async composition with depth first traversal. In
818−-- a left to right 'Semigroup' composition it tries to yield elements from the
819−-- left stream as long as it can, but it can run the right stream in parallel
820−-- if it needs to, based on demand. The right stream can be run if the left
821−-- stream blocks on IO or cannot produce elements fast enough for the consumer.
822−--
823−-- @
824−-- main = ('toList' . 'asyncly' $ (fromFoldable [1,2]) \<> (fromFoldable [3,4])) >>= print
825−-- @
826−-- @
827−-- [1,2,3,4]
828−-- @
829−--
830−-- Any exceptions generated by a constituent stream are propagated to the
831−-- output stream. The output and exceptions from a single stream are guaranteed
832−-- to arrive in the same order in the resulting stream as they were generated
833−-- in the input stream. However, the relative ordering of elements from
834−-- different streams in the resulting stream can vary depending on scheduling
835−-- and generation delays.
836−--
837−-- Similarly, the monad instance of 'AsyncT' /may/ run each iteration
838−-- concurrently based on demand. More concurrent iterations are started only
839−-- if the previous iterations are not able to produce enough output for the
840−-- consumer.
841−--
842−-- @
843−-- import "Streamly"
844−-- import qualified "Streamly.Prelude" as S
845−-- import Control.Concurrent
846−--
847−-- main = 'runStream' . 'asyncly' $ do
848−-- n <- return 3 \<\> return 2 \<\> return 1
849−-- S.once $ do
850−-- threadDelay (n * 1000000)
851−-- myThreadId >>= \\tid -> putStrLn (show tid ++ ": Delay " ++ show n)
852−-- @
853−-- @
854−-- ThreadId 40: Delay 1
855−-- ThreadId 39: Delay 2
856−-- ThreadId 38: Delay 3
857−-- @
858−--
859−-- All iterations may run in the same thread if they do not block.
860−--
861−-- Note that async composition with depth first traversal can be used to
862−-- combine infinite number of streams as it explores only a bounded number of
863−-- streams at a time.
864−--
865−-- @since 0.1.0
866−newtype AsyncT m a = AsyncT {getAsyncT :: C.Stream m a}
867− deriving (Functor, MonadTrans)
868−
869−instance IsStream AsyncT where
870− toStream = getAsyncT
871− fromStream = AsyncT
872−
873− {-# INLINE consM #-}
874− {-# SPECIALIZE consM :: IO a -> AsyncT IO a -> AsyncT IO a #-}
875− consM m r = fromStream $ CC.consMAsync m (toStream r)
876−
877− {-# INLINE (|:) #-}
878− {-# SPECIALIZE (|:) :: IO a -> AsyncT IO a -> AsyncT IO a #-}
879− (|:) = consM
880−
881−------------------------------------------------------------------------------
882−-- Semigroup
883−------------------------------------------------------------------------------
884−
885−-- | Polymorphic version of the 'Semigroup' operation '<>' of 'AsyncT'.
886−-- Merges two streams possibly concurrently, preferring the
887−-- elements from the left one when available.
888−--
889−-- @since 0.2.0
890−{-# INLINE async #-}
891−async :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
892−async m1 m2 = fromStream $ CC.async (toStream m1) (toStream m2)
893−
894−-- | Same as 'async'.
895−--
896−-- @since 0.1.0
897−{-# DEPRECATED (<|) "Please use 'async' instead." #-}
898−{-# INLINE (<|) #-}
899−(<|) :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
900−(<|) = async
901−
902−instance MonadAsync m => Semigroup (AsyncT m a) where
903− (<>) = async
904−
905−------------------------------------------------------------------------------
906−-- Monoid
907−------------------------------------------------------------------------------
908−
909−instance MonadAsync m => Monoid (AsyncT m a) where
910− mempty = nil
911− mappend = (<>)
912−
913−------------------------------------------------------------------------------
914−-- Monad
915−------------------------------------------------------------------------------
916−
917−{-# INLINE parbind #-}
918−parbind
919− :: (forall c. C.Stream m c -> C.Stream m c -> C.Stream m c)
920− -> C.Stream m a
921− -> (a -> C.Stream m b)
922− -> C.Stream m b
923−parbind par m f = go m
924− where
925− go (C.Stream g) =
926− C.Stream $ \ctx stp sng yld ->
927− let run x = (C.runStream x) ctx stp sng yld
928− single a = run $ f a
929− yield a r = run $ f a `par` go r
930− in g Nothing stp single yield
931−
932−instance MonadAsync m => Monad (AsyncT m) where
933− return = pure
934− (AsyncT m) >>= f = AsyncT $ parbind CC.async m (getAsyncT . f)
935−
936−------------------------------------------------------------------------------
937−-- Other instances
938−------------------------------------------------------------------------------
939−
940−MONAD_APPLICATIVE_INSTANCE(AsyncT,MONADPARALLEL)
941−MONAD_COMMON_INSTANCES(AsyncT, MONADPARALLEL)
942−
943−------------------------------------------------------------------------------
944−-- WAsyncT
945−------------------------------------------------------------------------------
946−
947−-- | Wide async composition or async composition with breadth first traversal.
948−-- The Semigroup instance of 'WAsyncT' concurrently /traverses/ the composed
949−-- streams using a depth first travesal or in a round robin fashion, yielding
950−-- elements from both streams alternately.
951−--
952−-- @
953−-- main = ('toList' . 'wAsyncly' $ (fromFoldable [1,2]) \<> (fromFoldable [3,4])) >>= print
954−-- @
955−-- @
956−-- [1,3,2,4]
957−-- @
958−--
959−-- Any exceptions generated by a constituent stream are propagated to the
960−-- output stream. The output and exceptions from a single stream are guaranteed
961−-- to arrive in the same order in the resulting stream as they were generated
962−-- in the input stream. However, the relative ordering of elements from
963−-- different streams in the resulting stream can vary depending on scheduling
964−-- and generation delays.
965−--
966−-- Similarly, the 'Monad' instance of 'WAsyncT' runs /all/ iterations fairly
967−-- concurrently using a round robin scheduling.
968−--
969−-- @
970−-- import "Streamly"
971−-- import qualified "Streamly.Prelude" as S
972−-- import Control.Concurrent
973−--
974−-- main = 'runStream' . 'wAsyncly' $ do
975−-- n <- return 3 \<\> return 2 \<\> return 1
976−-- S.once $ do
977−-- threadDelay (n * 1000000)
978−-- myThreadId >>= \\tid -> putStrLn (show tid ++ ": Delay " ++ show n)
979−-- @
980−-- @
981−-- ThreadId 40: Delay 1
982−-- ThreadId 39: Delay 2
983−-- ThreadId 38: Delay 3
984−-- @
985−--
986−-- Unlike 'AsyncT' all iterations are guaranteed to run fairly
987−-- concurrently, unconditionally.
988−--
989−-- Note that async composition with breadth first traversal can only combine a
990−-- finite number of streams as it needs to retain state for each unfinished
991−-- stream.
992−--
993−-- @since 0.2.0
994−newtype WAsyncT m a = WAsyncT {getWAsyncT :: C.Stream m a}
995− deriving (Functor, MonadTrans)
996−
997−instance IsStream WAsyncT where
998− toStream = getWAsyncT
999− fromStream = WAsyncT
1000−
1001− {-# INLINE consM #-}
1002− {-# SPECIALIZE consM :: IO a -> WAsyncT IO a -> WAsyncT IO a #-}
1003− consM m r = fromStream $ CC.consMWAsync m (toStream r)
1004−
1005− {-# INLINE (|:) #-}
1006− {-# SPECIALIZE (|:) :: IO a -> WAsyncT IO a -> WAsyncT IO a #-}
1007− (|:) = consM
1008−
1009−------------------------------------------------------------------------------
1010−-- Semigroup
1011−------------------------------------------------------------------------------
1012−
1013−-- | Polymorphic version of the 'Semigroup' operation '<>' of 'WAsyncT'.
1014−-- Merges two streams concurrently choosing elements from both fairly.
1015−--
1016−-- @since 0.2.0
1017−{-# INLINE wAsync #-}
1018−wAsync :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
1019−wAsync m1 m2 = fromStream $ CC.wAsync (toStream m1) (toStream m2)
1020−
1021−instance MonadAsync m => Semigroup (WAsyncT m a) where
1022− (<>) = wAsync
1023−
1024−------------------------------------------------------------------------------
1025−-- Monoid
1026−------------------------------------------------------------------------------
1027−
1028−instance MonadAsync m => Monoid (WAsyncT m a) where
1029− mempty = nil
1030− mappend = (<>)
1031−
1032−------------------------------------------------------------------------------
1033−-- Monad
1034−------------------------------------------------------------------------------
1035−
1036−instance MonadAsync m => Monad (WAsyncT m) where
1037− return = pure
1038− (WAsyncT m) >>= f =
1039− WAsyncT $ parbind CC.wAsync m (getWAsyncT . f)
1040−
1041−------------------------------------------------------------------------------
1042−-- Other instances
1043−------------------------------------------------------------------------------
1044−
1045−MONAD_APPLICATIVE_INSTANCE(WAsyncT,MONADPARALLEL)
1046−MONAD_COMMON_INSTANCES(WAsyncT, MONADPARALLEL)
1047−
1048−------------------------------------------------------------------------------
1049−-- ParallelT
1050−------------------------------------------------------------------------------
1051−
1052−-- | Async composition with simultaneous traversal of all streams.
1053−--
1054−-- The Semigroup instance of 'ParallelT' concurrently /merges/ two streams,
1055−-- running both strictly concurrently and yielding elements from both streams
1056−-- as they arrive. When multiple streams are combined using 'ParallelT' each
1057−-- one is evaluated in its own thread and the results produced are presented in
1058−-- the combined stream on a first come first serve basis.
1059−--
1060−-- 'AsyncT' and 'WAsyncT' are /concurrent lookahead streams/ each with a
1061−-- specific type of consumption pattern (depth first or breadth first). Since
1062−-- they are lookahead, they may introduce certain default latency in starting
1063−-- more concurrent tasks for efficiency reasons or may put a default limitation
1064−-- on the resource consumption (e.g. number of concurrent threads for
1065−-- lookahead). If we look at the implementation detail, they both can share a
1066−-- pool of worker threads to evaluate the streams in the desired pattern and at
1067−-- the desired rate. However, 'ParallelT' uses a separate runtime thread to
1068−-- evaluate each stream.
1069−--
1070−-- 'WAsyncT' is similar to 'ParallelT', as both of them evaluate the
1071−-- constituent streams fairly in a round robin fashion.
1072−-- However, the key difference is that 'WAsyncT' is lazy or pull driven
1073−-- whereas 'ParallelT' is strict or push driven. 'ParallelT' immediately
1074−-- starts concurrent evaluation of both the streams (in separate threads) and
1075−-- later picks the results whereas 'WAsyncT' may wait for a certain latency
1076−-- threshold before initiating concurrent evaluation of the next stream. The
1077−-- concurrent scheduling of the next stream or the degree of concurrency is
1078−-- driven by the feedback from the consumer. In case of 'ParallelT' each stream
1079−-- is evaluated in a separate thread and results are /pushed/ to a shared
1080−-- output buffer, the evaluation rate is controlled by blocking when the buffer
1081−-- is full.
1082−--
1083−-- Concurrent lookahead streams are generally more efficient than
1084−-- 'ParallelT' and can work pretty efficiently even for smaller tasks because
1085−-- they do not necessarily use a separate thread for each task. So they should
1086−-- be preferred over 'ParallelT' especially when efficiency is a concern and
1087−-- simultaneous strict evaluation is not a requirement. 'ParallelT' is useful
1088−-- for cases when the streams are required to be evaluated simultaneously
1089−-- irrespective of how the consumer consumes them e.g. when we want to race
1090−-- two tasks and want to start both strictly at the same time or if we have
1091−-- timers in the parallel tasks and our results depend on the timers being
1092−-- started at the same time. We can say that 'ParallelT' is almost the same
1093−-- (modulo some implementation differences) as 'WAsyncT' when the latter is
1094−-- used with unlimited lookahead and zero latency in initiating lookahead.
1095−--
1096−-- @
1097−-- main = ('toList' . 'parallely' $ (fromFoldable [1,2]) \<> (fromFoldable [3,4])) >>= print
1098−-- @
1099−-- @
1100−-- [1,3,2,4]
1101−-- @
1102−--
1103−-- When streams with more than one element are merged, it yields whichever
1104−-- stream yields first without any bias, unlike the 'Async' style streams.
1105−--
1106−-- Any exceptions generated by a constituent stream are propagated to the
1107−-- output stream. The output and exceptions from a single stream are guaranteed
1108−-- to arrive in the same order in the resulting stream as they were generated
1109−-- in the input stream. However, the relative ordering of elements from
1110−-- different streams in the resulting stream can vary depending on scheduling
1111−-- and generation delays.
1112−--
1113−-- Similarly, the 'Monad' instance of 'ParallelT' runs /all/ iterations
1114−-- of the loop concurrently.
1115−--
1116−-- @
1117−-- import "Streamly"
1118−-- import qualified "Streamly.Prelude" as S
1119−-- import Control.Concurrent
1120−--
1121−-- main = 'runStream' . 'parallely' $ do
1122−-- n <- return 3 \<\> return 2 \<\> return 1
1123−-- S.once $ do
1124−-- threadDelay (n * 1000000)
1125−-- myThreadId >>= \\tid -> putStrLn (show tid ++ ": Delay " ++ show n)
1126−-- @
1127−-- @
1128−-- ThreadId 40: Delay 1
1129−-- ThreadId 39: Delay 2
1130−-- ThreadId 38: Delay 3
1131−-- @
1132−--
1133−-- Note that parallel composition can only combine a finite number of
1134−-- streams as it needs to retain state for each unfinished stream.
1135−--
1136−-- @since 0.1.0
1137−newtype ParallelT m a = ParallelT {getParallelT :: C.Stream m a}
1138− deriving (Functor, MonadTrans)
1139−
1140−instance IsStream ParallelT where
1141− toStream = getParallelT
1142− fromStream = ParallelT
1143−
1144− {-# INLINE consM #-}
1145− {-# SPECIALIZE consM :: IO a -> ParallelT IO a -> ParallelT IO a #-}
1146− consM m r = fromStream $ CC.consMParallel m (toStream r)
1147−
1148− {-# INLINE (|:) #-}
1149− {-# SPECIALIZE (|:) :: IO a -> ParallelT IO a -> ParallelT IO a #-}
1150− (|:) = consM
1151−
1152−------------------------------------------------------------------------------
1153−-- Semigroup
1154−------------------------------------------------------------------------------
1155−
1156−-- | Polymorphic version of the 'Semigroup' operation '<>' of 'ParallelT'
1157−-- Merges two streams concurrently.
1158−--
1159−-- @since 0.2.0
1160−{-# INLINE parallel #-}
1161−parallel :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
1162−parallel m1 m2 = fromStream $ CC.parallel (toStream m1) (toStream m2)
1163−
1164−instance MonadAsync m => Semigroup (ParallelT m a) where
1165− (<>) = parallel
1166−
1167−------------------------------------------------------------------------------
1168−-- Monoid
1169−------------------------------------------------------------------------------
1170−
1171−instance MonadAsync m => Monoid (ParallelT m a) where
1172− mempty = nil
1173− mappend = (<>)
1174−
1175−------------------------------------------------------------------------------
1176−-- Monad
1177−------------------------------------------------------------------------------
1178−
1179−instance MonadAsync m => Monad (ParallelT m) where
1180− return = pure
1181− (ParallelT m) >>= f = ParallelT $ parbind CC.parallel m (getParallelT . f)
1182−
1183−------------------------------------------------------------------------------
1184−-- Other instances
1185−------------------------------------------------------------------------------
1186−
1187−MONAD_APPLICATIVE_INSTANCE(ParallelT,MONADPARALLEL)
1188−MONAD_COMMON_INSTANCES(ParallelT, MONADPARALLEL)
1189−
1190−------------------------------------------------------------------------------
1191−-- Serially Zipping Streams
1192−------------------------------------------------------------------------------
1193−
1194−-- | The applicative instance of 'ZipSerialM' zips a number of streams serially
1195−-- i.e. it produces one element from each stream serially and then zips all
1196−-- those elements.
1197−--
1198−-- @
1199−-- main = (toList . 'zipSerially' $ (,,) \<$\> s1 \<*\> s2 \<*\> s3) >>= print
1200−-- where s1 = fromFoldable [1, 2]
1201−-- s2 = fromFoldable [3, 4]
1202−-- s3 = fromFoldable [5, 6]
1203−-- @
1204−-- @
1205−-- [(1,3,5),(2,4,6)]
1206−-- @
1207−--
1208−-- The 'Semigroup' instance of this type works the same way as that of
1209−-- 'SerialT'.
1210−--
1211−-- @since 0.2.0
1212−newtype ZipSerialM m a = ZipSerialM {getZipSerialM :: C.Stream m a}
1213− deriving (Functor, Semigroup, Monoid)
1214−
1215−-- |
1216−-- @since 0.1.0
1217−{-# DEPRECATED ZipStream "Please use 'ZipSerialM' instead." #-}
1218−type ZipStream = ZipSerialM
1219−
1220−instance IsStream ZipSerialM where
1221− toStream = getZipSerialM
1222− fromStream = ZipSerialM
1223−
1224− {-# INLINE consM #-}
1225− {-# SPECIALIZE consM :: IO a -> ZipSerialM IO a -> ZipSerialM IO a #-}
1226− consM :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a
1227− consM = consMSerial
1228−
1229− {-# INLINE (|:) #-}
1230− {-# SPECIALIZE (|:) :: IO a -> ZipSerialM IO a -> ZipSerialM IO a #-}
1231− (|:) :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a
1232− (|:) = consMSerial
1233−
1234−instance Monad m => Applicative (ZipSerialM m) where
1235− pure = ZipSerialM . C.repeat
1236− m1 <*> m2 = fromStream $ C.zipWith id (toStream m1) (toStream m2)
1237−
1238−------------------------------------------------------------------------------
1239−-- Parallely Zipping Streams
1240−------------------------------------------------------------------------------
1241−
1242−-- | Like 'ZipSerialM' but zips in parallel, it generates all the elements to
1243−-- be zipped concurrently.
1244−--
1245−-- @
1246−-- main = (toList . 'zipAsyncly' $ (,,) \<$\> s1 \<*\> s2 \<*\> s3) >>= print
1247−-- where s1 = fromFoldable [1, 2]
1248−-- s2 = fromFoldable [3, 4]
1249−-- s3 = fromFoldable [5, 6]
1250−-- @
1251−-- @
1252−-- [(1,3,5),(2,4,6)]
1253−-- @
1254−--
1255−-- The 'Semigroup' instance of this type works the same way as that of
1256−-- 'SerialT'.
1257−--
1258−-- @since 0.2.0
1259−newtype ZipAsyncM m a = ZipAsyncM {getZipAsyncM :: C.Stream m a}
1260− deriving (Functor, Semigroup, Monoid)
1261−
1262−instance IsStream ZipAsyncM where
1263− toStream = getZipAsyncM
1264− fromStream = ZipAsyncM
1265−
1266− {-# INLINE consM #-}
1267− {-# SPECIALIZE consM :: IO a -> ZipAsyncM IO a -> ZipAsyncM IO a #-}
1268− consM :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a
1269− consM = consMSerial
1270−
1271− {-# INLINE (|:) #-}
1272− {-# SPECIALIZE (|:) :: IO a -> ZipAsyncM IO a -> ZipAsyncM IO a #-}
1273− (|:) :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a
1274− (|:) = consMSerial
1275−
1276−instance MonadAsync m => Applicative (ZipAsyncM m) where
1277− pure = ZipAsyncM . C.repeat
1278− m1 <*> m2 = fromStream $ CC.zipAsyncWith id (toStream m1) (toStream m2)
1279−
1280−-------------------------------------------------------------------------------
1281−-- Type adapting combinators
1282−-------------------------------------------------------------------------------
1283−
1284−-- | Adapt any specific stream type to any other specific stream type.
1285−--
1286−-- @since 0.1.0
1287−adapt :: (IsStream t1, IsStream t2) => t1 m a -> t2 m a
1288−adapt = fromStream . toStream
1289−
1290−-- | Fix the type of a polymorphic stream as 'SerialT'.
1291−--
1292−-- @since 0.1.0
1293−serially :: IsStream t => SerialT m a -> t m a
1294−serially = adapt
1295−
1296−-- | Fix the type of a polymorphic stream as 'WSerialT'.
1297−--
1298−-- @since 0.2.0
1299−wSerially :: IsStream t => WSerialT m a -> t m a
1300−wSerially = adapt
1301−
1302−-- | Same as 'wSerially'.
1303−--
1304−-- @since 0.1.0
1305−{-# DEPRECATED interleaving "Please use wSerially instead." #-}
1306−interleaving :: IsStream t => WSerialT m a -> t m a
1307−interleaving = wSerially
1308−
1309−-- | Fix the type of a polymorphic stream as 'AheadT'.
1310−--
1311−-- @since 0.3.0
1312−aheadly :: IsStream t => AheadT m a -> t m a
1313−aheadly = adapt
1314−
1315−-- | Fix the type of a polymorphic stream as 'AsyncT'.
1316−--
1317−-- @since 0.1.0
1318−asyncly :: IsStream t => AsyncT m a -> t m a
1319−asyncly = adapt
1320−
1321−-- | Fix the type of a polymorphic stream as 'WAsyncT'.
1322−--
1323−-- @since 0.2.0
1324−wAsyncly :: IsStream t => WAsyncT m a -> t m a
1325−wAsyncly = adapt
1326−
1327−-- | Fix the type of a polymorphic stream as 'ParallelT'.
1328−--
1329−-- @since 0.1.0
1330−parallely :: IsStream t => ParallelT m a -> t m a
1331−parallely = adapt
1332−
1333−-- | Fix the type of a polymorphic stream as 'ZipSerialM'.
1334−--
1335−-- @since 0.2.0
1336−zipSerially :: IsStream t => ZipSerialM m a -> t m a
1337−zipSerially = adapt
1338−
1339−-- | Same as 'zipSerially'.
1340−--
1341−-- @since 0.1.0
1342−{-# DEPRECATED zipping "Please use zipSerially instead." #-}
1343−zipping :: IsStream t => ZipSerialM m a -> t m a
1344−zipping = zipSerially
1345−
1346−-- | Fix the type of a polymorphic stream as 'ZipAsyncM'.
1347−--
1348−-- @since 0.2.0
1349−zipAsyncly :: IsStream t => ZipAsyncM m a -> t m a
1350−zipAsyncly = adapt
1351−
1352−-- | Same as 'zipAsyncly'.
1353−--
1354−-- @since 0.1.0
1355−{-# DEPRECATED zippingAsync "Please use zipAsyncly instead." #-}
1356−zippingAsync :: IsStream t => ZipAsyncM m a -> t m a
1357−zippingAsync = zipAsyncly
1358−
1359−-------------------------------------------------------------------------------
1360−-- Running Streams, convenience functions specialized to types
1361−-------------------------------------------------------------------------------
1362−
1363−-- | Same as @runStream@.
1364−--
1365−-- @since 0.1.0
1366−{-# DEPRECATED runStreamT "Please use runStream instead." #-}
1367−runStreamT :: Monad m => SerialT m a -> m ()
1368−runStreamT = runStream
1369−
1370−-- | Same as @runStream . wSerially@.
1371−--
1372−-- @since 0.1.0
1373−{-# DEPRECATED runInterleavedT "Please use 'runStream . interleaving' instead." #-}
1374−runInterleavedT :: Monad m => InterleavedT m a -> m ()
1375−runInterleavedT = runStream . wSerially
1376−
1377−-- | Same as @runStream . asyncly@.
1378−--
1379−-- @since 0.1.0
1380−{-# DEPRECATED runAsyncT "Please use 'runStream . asyncly' instead." #-}
1381−runAsyncT :: Monad m => AsyncT m a -> m ()
1382−runAsyncT = runStream . asyncly
1383−
1384−-- | Same as @runStream . parallely@.
1385−--
1386−-- @since 0.1.0
1387−{-# DEPRECATED runParallelT "Please use 'runStream . parallely' instead." #-}
1388−runParallelT :: Monad m => ParallelT m a -> m ()
1389−runParallelT = runStream . parallely
1390−
1391−-- | Same as @runStream . zipping@.
1392−--
1393−-- @since 0.1.0
1394−{-# DEPRECATED runZipStream "Please use 'runStream . zipSerially instead." #-}
1395−runZipStream :: Monad m => ZipSerialM m a -> m ()
1396−runZipStream = runStream . zipSerially
1397−
1398−-- | Same as @runStream . zippingAsync@.
1399−--
1400−-- @since 0.1.0
1401−{-# DEPRECATED runZipAsync "Please use 'runStream . zipAsyncly instead." #-}
1402−runZipAsync :: Monad m => ZipAsyncM m a -> m ()
1403−runZipAsync = runStream . zipAsyncly
1404−
1405−------------------------------------------------------------------------------
1406−-- IO Streams
1407−------------------------------------------------------------------------------
1408−
1409−-- | A serial IO stream of elements of type @a@. See 'SerialT' documentation
1410−-- for more details.
1411−--
1412−-- @since 0.2.0
1413−type Serial a = SerialT IO a
1414−
1415−-- | An interleaving serial IO stream of elements of type @a@. See 'WSerialT'
1416−-- documentation for more details.
1417−--
1418−-- @since 0.2.0
1419−type WSerial a = WSerialT IO a
1420−
1421−-- | A serial IO stream of elements of type @a@ with concurrent lookahead. See
1422−-- 'AheadT' documentation for more details.
1423−--
1424−-- @since 0.3.0
1425−type Ahead a = AheadT IO a
1426−
1427−-- | A demand driven left biased parallely composing IO stream of elements of
1428−-- type @a@. See 'AsyncT' documentation for more details.
1429−--
1430−-- @since 0.2.0
1431−type Async a = AsyncT IO a
1432−
1433−-- | A round robin parallely composing IO stream of elements of type @a@.
1434−-- See 'WAsyncT' documentation for more details.
1435−--
1436−-- @since 0.2.0
1437−type WAsync a = WAsyncT IO a
1438−
1439−-- | A parallely composing IO stream of elements of type @a@.
1440−-- See 'ParallelT' documentation for more details.
1441−--
1442−-- @since 0.2.0
1443−type Parallel a = ParallelT IO a
1444−
1445−-- | An IO stream whose applicative instance zips streams serially.
1446−--
1447−-- @since 0.2.0
1448−type ZipSerial a = ZipSerialM IO a
1449−
1450−-- | An IO stream whose applicative instance zips streams wAsyncly.
1451−--
1452−-- @since 0.2.0
1453−type ZipAsync a = ZipAsyncM IO a
1454−
1455−------------------------------------------------------------------------------
1456−-- Fold Utilities
1457−------------------------------------------------------------------------------
1458−
1459−-- | A variant of 'Data.Foldable.fold' that allows you to fold a 'Foldable'
1460−-- container of streams using the specified stream sum operation.
1461−--
1462−-- @foldWith 'async' $ map return [1..3]@
1463−--
1464−-- @since 0.1.0
1465−{-# INLINABLE foldWith #-}
1466−foldWith :: (IsStream t, Foldable f)
1467− => (t m a -> t m a -> t m a) -> f (t m a) -> t m a
1468−foldWith f = foldr f nil
1469−
1470−-- | A variant of 'foldMap' that allows you to map a monadic streaming action
1471−-- on a 'Foldable' container and then fold it using the specified stream sum
1472−-- operation.
1473−--
1474−-- @foldMapWith 'async' return [1..3]@
1475−--
1476−-- @since 0.1.0
1477−{-# INLINABLE foldMapWith #-}
1478−foldMapWith :: (IsStream t, Foldable f)
1479− => (t m b -> t m b -> t m b) -> (a -> t m b) -> f a -> t m b
1480−foldMapWith f g = foldr (f . g) nil
1481−
1482−-- | Like 'foldMapWith' but with the last two arguments reversed i.e. the
1483−-- monadic streaming function is the last argument.
1484−--
1485−-- @since 0.1.0
1486−{-# INLINABLE forEachWith #-}
1487−forEachWith :: (IsStream t, Foldable f)
1488− => (t m b -> t m b -> t m b) -> f a -> (a -> t m b) -> t m b
1489−forEachWith f xs g = foldr (f . g) nil xs
src/Streamly/Streams/Ahead.hsnew file+373
@@ -0,0 +1,373 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE ConstraintKinds #-}
3+{-# LANGUAGE FlexibleContexts #-}
4+{-# LANGUAGE FlexibleInstances #-}
5+{-# LANGUAGE GeneralizedNewtypeDeriving#-}
6+{-# LANGUAGE InstanceSigs #-}
7+{-# LANGUAGE MultiParamTypeClasses #-}
8+{-# LANGUAGE StandaloneDeriving #-}
9+{-# LANGUAGE UndecidableInstances #-} -- XXX
10+
11+-- |
12+-- Module : Streamly.Streams.Ahead
13+-- Copyright : (c) 2017 Harendra Kumar
14+--
15+-- License : BSD3
16+-- Maintainer : harendra.kumar@gmail.com
17+-- Stability : experimental
18+-- Portability : GHC
19+--
20+--
21+module Streamly.Streams.Ahead
22+ (
23+ AheadT
24+ , Ahead
25+ , aheadly
26+ , ahead
27+ )
28+where
29+
30+import Control.Monad (ap)
31+import Control.Monad.Base (MonadBase(..), liftBaseDefault)
32+import Control.Monad.Catch (MonadThrow, throwM)
33+-- import Control.Monad.Error.Class (MonadError(..))
34+import Control.Monad.IO.Class (MonadIO(..))
35+import Control.Monad.Reader.Class (MonadReader(..))
36+import Control.Monad.State.Class (MonadState(..))
37+import Control.Monad.Trans.Class (MonadTrans(lift))
38+import Data.Atomics (atomicModifyIORefCAS_)
39+import Data.Heap (Heap, Entry(..))
40+import Data.IORef (IORef)
41+import Data.Maybe (fromJust)
42+import Data.Semigroup (Semigroup(..))
43+
44+import qualified Data.Heap as H
45+
46+import Streamly.Streams.SVar (fromSVar)
47+import Streamly.Streams.CPS
48+import Streamly.SVar
49+
50+#ifdef DIAGNOSTICS
51+import Control.Monad (when)
52+import Data.IORef (writeIORef, readIORef)
53+#endif
54+
55+#include "Instances.hs"
56+
57+-------------------------------------------------------------------------------
58+-- Ahead
59+-------------------------------------------------------------------------------
60+
61+-- Lookahead streams can execute multiple tasks concurrently, ahead of time,
62+-- but always serve them in the same order as they appear in the stream. To
63+-- implement lookahead streams efficiently we assign a sequence number to each
64+-- task when the task is picked up for execution. When the task finishes, the
65+-- output is tagged with the same sequence number and we rearrange the outputs
66+-- in sequence based on that number.
67+--
68+-- To explain the mechanism imagine that the current task at the head of the
69+-- stream has a "token" to yield to the outputQueue. The ownership of the token
70+-- is determined by the current sequence number is maintained in outputHeap.
71+-- Sequence number is assigned when a task is queued. When a thread dequeues a
72+-- task it picks up the sequence number as well and when the output is ready it
73+-- uses the sequence number to queue the output to the outputQueue.
74+--
75+-- The thread with current sequence number sends the output directly to the
76+-- outputQueue. Other threads push the output to the outputHeap. When the task
77+-- being queued on the heap is a stream of many elements we evaluate only the
78+-- first element and keep the rest of the unevaluated computation in the heap.
79+-- When such a task gets the "token" for outputQueue it evaluates and directly
80+-- yields all the elements to the outputQueue without checking for the
81+-- "token".
82+--
83+-- Note that no two outputs in the heap can have the same sequence numbers and
84+-- therefore we do not need a stable heap. We have also separated the buffer
85+-- for the current task (outputQueue) and the pending tasks (outputHeap) so
86+-- that the pending tasks cannot interfere with the current task. Note that for
87+-- a single task just the outputQueue is enough and for the case of many
88+-- threads just a heap is good enough. However we balance between these two
89+-- cases, so that both are efficient.
90+--
91+-- For bigger streams it may make sense to have separate buffers for each
92+-- stream. However, for singleton streams this may become inefficient. However,
93+-- if we do not have separate buffers, then the streams that come later in
94+-- sequence may hog the buffer, hindering the streams that are ahead. For this
95+-- reason we have a single element buffer limitation for the streams being
96+-- executed in advance.
97+--
98+-- This scheme works pretty efficiently with less than 40% extra overhead
99+-- compared to the Async streams where we do not have any kind of sequencing of
100+-- the outputs. It is especially devised so that we are most efficient when we
101+-- have short tasks and need just a single thread. Also when a thread yields
102+-- many items it can hold lockfree access to the outputQueue and do it
103+-- efficiently.
104+--
105+-- XXX Maybe we can start the ahead threads at a lower cpu and IO priority so
106+-- that they do not hog the resources and hinder the progress of the threads in
107+-- front of them.
108+
109+-- Left associated ahead expressions are expensive. We start a new SVar for
110+-- each left associative expression. The queue is used only for right
111+-- associated expression, we queue the right expression and execute the left.
112+-- Thererefore the queue never has more than on item in it.
113+
114+workLoopAhead :: MonadIO m
115+ => SVar Stream m a
116+ -> IORef ([Stream m a], Int)
117+ -> IORef (Heap (Entry Int (AheadHeapEntry Stream m a)) , Int)
118+ -> m ()
119+workLoopAhead sv q heap = runHeap
120+
121+ where
122+
123+ toHeap seqNo ent = do
124+ hp <- liftIO $ atomicModifyIORefCAS heap $ \(h, snum) ->
125+ ((H.insert (Entry seqNo ent) h, snum), h)
126+ if H.size hp <= maxHeap
127+ then runHeap
128+ else liftIO $ sendStop sv
129+
130+ singleToHeap seqNo a = toHeap seqNo (AheadEntryPure a)
131+ yieldToHeap seqNo a r = toHeap seqNo (AheadEntryStream (a `cons` r))
132+
133+ singleOutput seqNo a = do
134+ continue <- liftIO $ send sv (ChildYield a)
135+ if continue
136+ then runQueueToken seqNo
137+ else liftIO $ do
138+ atomicModifyIORefCAS_ heap $ \(h, _) -> (h, seqNo + 1)
139+ sendStop sv
140+
141+ yieldOutput seqNo a r = do
142+ continue <- liftIO $ send sv (ChildYield a)
143+ if continue
144+ then (runStream r) (Just sv) (runQueueToken seqNo)
145+ (singleOutput seqNo)
146+ (yieldOutput seqNo)
147+ else liftIO $ do
148+ atomicModifyIORefCAS_ heap $ \(h, _) ->
149+ (H.insert (Entry seqNo (AheadEntryStream r)) h, seqNo)
150+ sendStop sv
151+
152+ {-# INLINE runQueueToken #-}
153+ runQueueToken prevSeqNo = do
154+ work <- dequeueAhead q
155+ case work of
156+ Nothing -> do
157+ liftIO $ atomicModifyIORefCAS_ heap $ \(h, _) ->
158+ (h, prevSeqNo + 1)
159+ runHeap
160+ Just (m, seqNo) -> do
161+ if seqNo == prevSeqNo + 1
162+ then
163+ (runStream m) (Just sv) (runQueueToken seqNo)
164+ (singleOutput seqNo)
165+ (yieldOutput seqNo)
166+ else do
167+ liftIO $ atomicModifyIORefCAS_ heap $ \(h, _) ->
168+ (h, prevSeqNo + 1)
169+ (runStream m) (Just sv) runHeap
170+ (singleToHeap seqNo)
171+ (yieldToHeap seqNo)
172+ runQueueNoToken = do
173+ work <- dequeueAhead q
174+ case work of
175+ Nothing -> runHeap
176+ Just (m, seqNo) -> do
177+ if seqNo == 0
178+ then
179+ (runStream m) (Just sv) (runQueueToken seqNo)
180+ (singleOutput seqNo)
181+ (yieldOutput seqNo)
182+ else
183+ (runStream m) (Just sv) runHeap
184+ (singleToHeap seqNo)
185+ (yieldToHeap seqNo)
186+
187+ {-# NOINLINE runHeap #-}
188+ runHeap = do
189+#ifdef DIAGNOSTICS
190+ liftIO $ do
191+ maxHp <- readIORef (maxHeapSize sv)
192+ (hp, _) <- readIORef heap
193+ when (H.size hp > maxHp) $ writeIORef (maxHeapSize sv) (H.size hp)
194+#endif
195+ ent <- liftIO $ dequeueFromHeap heap
196+ case ent of
197+ Nothing -> do
198+ done <- queueEmptyAhead q
199+ if done
200+ then liftIO $ sendStop sv
201+ else runQueueNoToken
202+ Just (Entry seqNo hent) -> do
203+ case hent of
204+ AheadEntryPure a -> singleOutput seqNo a
205+ AheadEntryStream r ->
206+ (runStream r) (Just sv) (runQueueToken seqNo)
207+ (singleOutput seqNo)
208+ (yieldOutput seqNo)
209+
210+-------------------------------------------------------------------------------
211+-- WAhead
212+-------------------------------------------------------------------------------
213+
214+-- XXX To be implemented. Use a linked queue like WAsync and put back the
215+-- remaining computation at the back of the queue instead of the heap, and
216+-- increment the sequence number.
217+
218+-- The only difference between forkSVarAsync and this is that we run the left
219+-- computation without a shared SVar.
220+forkSVarAhead :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
221+forkSVarAhead m1 m2 = Stream $ \_ stp sng yld -> do
222+ sv <- newAheadVar (concurrently m1 m2) workLoopAhead
223+ (runStream (fromSVar sv)) Nothing stp sng yld
224+ where
225+ concurrently ma mb = Stream $ \svr stp sng yld -> do
226+ liftIO $ enqueue (fromJust svr) mb
227+ (runStream ma) Nothing stp sng yld
228+
229+{-# INLINE aheadS #-}
230+aheadS :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
231+aheadS m1 m2 = Stream $ \svr stp sng yld -> do
232+ case svr of
233+ Just sv | svarStyle sv == AheadVar -> do
234+ liftIO $ enqueue sv m2
235+ -- Always run the left side on a new SVar to avoid complexity in
236+ -- sequencing results. This means the left side cannot further
237+ -- split into more ahead computations on the same SVar.
238+ (runStream m1) Nothing stp sng yld
239+ _ -> runStream (forkSVarAhead m1 m2) Nothing stp sng yld
240+
241+-- | XXX we can implement it more efficienty by directly implementing instead
242+-- of combining streams using ahead.
243+{-# INLINE consMAhead #-}
244+consMAhead :: MonadAsync m => m a -> Stream m a -> Stream m a
245+consMAhead m r = once m `aheadS` r
246+
247+------------------------------------------------------------------------------
248+-- AheadT
249+------------------------------------------------------------------------------
250+
251+-- | Deep ahead composition or ahead composition with depth first traversal.
252+-- The semigroup composition of 'AheadT' appends streams in a depth first
253+-- manner just like 'SerialT' except that it can produce elements concurrently
254+-- ahead of time. It is like 'AsyncT' except that 'AsyncT' produces the output
255+-- as it arrives whereas 'AheadT' orders the output in the traversal order.
256+--
257+-- @
258+-- main = ('toList' . 'aheadly' $ (fromFoldable [1,2]) \<> (fromFoldable [3,4])) >>= print
259+-- @
260+-- @
261+-- [1,2,3,4]
262+-- @
263+--
264+-- Any exceptions generated by a constituent stream are propagated to the
265+-- output stream.
266+--
267+-- Similarly, the monad instance of 'AheadT' may run each iteration
268+-- concurrently ahead of time but presents the results in the same order as
269+-- 'SerialT'.
270+--
271+-- @
272+-- import "Streamly"
273+-- import qualified "Streamly.Prelude" as S
274+-- import Control.Concurrent
275+--
276+-- main = 'runStream' . 'aheadly' $ do
277+-- n <- return 3 \<\> return 2 \<\> return 1
278+-- S.once $ do
279+-- threadDelay (n * 1000000)
280+-- myThreadId >>= \\tid -> putStrLn (show tid ++ ": Delay " ++ show n)
281+-- @
282+-- @
283+-- ThreadId 40: Delay 1
284+-- ThreadId 39: Delay 2
285+-- ThreadId 38: Delay 3
286+-- @
287+--
288+-- All iterations may run in the same thread if they do not block.
289+--
290+-- Note that ahead composition with depth first traversal can be used to
291+-- combine infinite number of streams as it explores only a bounded number of
292+-- streams at a time.
293+--
294+-- @since 0.3.0
295+newtype AheadT m a = AheadT {getAheadT :: Stream m a}
296+ deriving (Functor, MonadTrans)
297+
298+-- | A serial IO stream of elements of type @a@ with concurrent lookahead. See
299+-- 'AheadT' documentation for more details.
300+--
301+-- @since 0.3.0
302+type Ahead a = AheadT IO a
303+
304+-- | Fix the type of a polymorphic stream as 'AheadT'.
305+--
306+-- @since 0.3.0
307+aheadly :: IsStream t => AheadT m a -> t m a
308+aheadly = adapt
309+
310+instance IsStream AheadT where
311+ toStream = getAheadT
312+ fromStream = AheadT
313+
314+ {-# INLINE consM #-}
315+ {-# SPECIALIZE consM :: IO a -> AheadT IO a -> AheadT IO a #-}
316+ consM m r = fromStream $ consMAhead m (toStream r)
317+
318+ {-# INLINE (|:) #-}
319+ {-# SPECIALIZE (|:) :: IO a -> AheadT IO a -> AheadT IO a #-}
320+ (|:) = consM
321+
322+------------------------------------------------------------------------------
323+-- Semigroup
324+------------------------------------------------------------------------------
325+
326+-- | Polymorphic version of the 'Semigroup' operation '<>' of 'AheadT'.
327+-- Merges two streams sequentially but with concurrent lookahead.
328+--
329+-- @since 0.3.0
330+{-# INLINE ahead #-}
331+ahead :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
332+ahead m1 m2 = fromStream $ aheadS (toStream m1) (toStream m2)
333+
334+instance MonadAsync m => Semigroup (AheadT m a) where
335+ (<>) = ahead
336+
337+------------------------------------------------------------------------------
338+-- Monoid
339+------------------------------------------------------------------------------
340+
341+instance MonadAsync m => Monoid (AheadT m a) where
342+ mempty = nil
343+ mappend = (<>)
344+
345+------------------------------------------------------------------------------
346+-- Monad
347+------------------------------------------------------------------------------
348+
349+{-# INLINE aheadbind #-}
350+aheadbind
351+ :: MonadAsync m
352+ => Stream m a
353+ -> (a -> Stream m b)
354+ -> Stream m b
355+aheadbind m f = go m
356+ where
357+ go (Stream g) =
358+ Stream $ \ctx stp sng yld ->
359+ let runIt x = (runStream x) ctx stp sng yld
360+ single a = runIt $ f a
361+ yield a r = runIt $ f a `aheadS` go r
362+ in g Nothing stp single yield
363+
364+instance MonadAsync m => Monad (AheadT m) where
365+ return = pure
366+ (AheadT m) >>= f = AheadT $ aheadbind m (getAheadT . f)
367+
368+------------------------------------------------------------------------------
369+-- Other instances
370+------------------------------------------------------------------------------
371+
372+MONAD_APPLICATIVE_INSTANCE(AheadT,MONADPARALLEL)
373+MONAD_COMMON_INSTANCES(AheadT, MONADPARALLEL)
src/Streamly/Streams/Async.hsnew file+570
@@ -0,0 +1,570 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE ConstraintKinds #-}
3+{-# LANGUAGE FlexibleContexts #-}
4+{-# LANGUAGE FlexibleInstances #-}
5+{-# LANGUAGE GeneralizedNewtypeDeriving#-}
6+{-# LANGUAGE InstanceSigs #-}
7+{-# LANGUAGE MultiParamTypeClasses #-}
8+{-# LANGUAGE StandaloneDeriving #-}
9+{-# LANGUAGE UndecidableInstances #-} -- XXX
10+
11+-- |
12+-- Module : Streamly.Streams.Async
13+-- Copyright : (c) 2017 Harendra Kumar
14+--
15+-- License : BSD3
16+-- Maintainer : harendra.kumar@gmail.com
17+-- Stability : experimental
18+-- Portability : GHC
19+--
20+--
21+module Streamly.Streams.Async
22+ (
23+ AsyncT
24+ , Async
25+ , asyncly
26+ , async
27+ , (<|) --deprecated
28+ , mkAsync
29+
30+ , WAsyncT
31+ , WAsync
32+ , wAsyncly
33+ , wAsync
34+ , runAsyncT -- deprecated
35+ )
36+where
37+
38+import Control.Monad (ap)
39+import Control.Monad.Base (MonadBase(..), liftBaseDefault)
40+import Control.Monad.Catch (MonadThrow, throwM)
41+import Control.Concurrent.MVar (newEmptyMVar)
42+-- import Control.Monad.Error.Class (MonadError(..))
43+import Control.Monad.IO.Class (MonadIO(..))
44+import Control.Monad.Reader.Class (MonadReader(..))
45+import Control.Monad.State.Class (MonadState(..))
46+import Control.Monad.Trans.Class (MonadTrans(lift))
47+import Data.Concurrent.Queue.MichaelScott (LinkedQueue, newQ, nullQ)
48+import Data.IORef (IORef, newIORef, readIORef)
49+import Data.Maybe (fromJust)
50+import Data.Semigroup (Semigroup(..))
51+
52+import qualified Data.Set as S
53+
54+import Streamly.Streams.SVar (fromSVar)
55+import Streamly.Streams.CPS
56+import Streamly.SVar
57+
58+#include "Instances.hs"
59+
60+-------------------------------------------------------------------------------
61+-- Async
62+-------------------------------------------------------------------------------
63+
64+{-# INLINE runStreamLIFO #-}
65+runStreamLIFO :: MonadIO m
66+ => SVar Stream m a -> IORef [Stream m a] -> Stream m a -> m () -> m ()
67+runStreamLIFO sv q m stop = runStream m (Just sv) stop single yield
68+ where
69+ single a = do
70+ res <- liftIO $ send sv (ChildYield a)
71+ if res then stop else liftIO $ sendStop sv
72+ yield a r = do
73+ res <- liftIO $ send sv (ChildYield a)
74+ if res
75+ then (runStream r) (Just sv) stop single yield
76+ else liftIO $ enqueueLIFO sv q r >> sendStop sv
77+
78+-------------------------------------------------------------------------------
79+-- WAsync
80+-------------------------------------------------------------------------------
81+
82+{-# INLINE runStreamFIFO #-}
83+runStreamFIFO :: MonadIO m
84+ => SVar Stream m a -> LinkedQueue (Stream m a) -> Stream m a -> m () -> m ()
85+runStreamFIFO sv q m stop = runStream m (Just sv) stop single yield
86+ where
87+ single a = do
88+ res <- liftIO $ send sv (ChildYield a)
89+ if res then stop else liftIO $ sendStop sv
90+ yield a r = do
91+ res <- liftIO $ send sv (ChildYield a)
92+ liftIO (enqueueFIFO sv q r)
93+ if res then stop else liftIO $ sendStop sv
94+
95+-------------------------------------------------------------------------------
96+-- SVar creation
97+-- This code belongs in SVar.hs but is kept here for perf reasons
98+-------------------------------------------------------------------------------
99+
100+-- XXX we have this function in this file because passing runStreamLIFO as a
101+-- function argument to this function results in a perf degradation of more
102+-- than 10%. Need to investigate what the root cause is.
103+-- Interestingly, the same thing does not make any difference for Ahead.
104+getLifoSVar :: MonadAsync m => IO (SVar Stream m a)
105+getLifoSVar = do
106+ outQ <- newIORef ([], 0)
107+ outQMv <- newEmptyMVar
108+ active <- newIORef 0
109+ wfw <- newIORef False
110+ running <- newIORef S.empty
111+ q <- newIORef []
112+#ifdef DIAGNOSTICS
113+ disp <- newIORef 0
114+ maxWrk <- newIORef 0
115+ maxOq <- newIORef 0
116+ maxHs <- newIORef 0
117+ maxWq <- newIORef 0
118+#endif
119+ let checkEmpty = null <$> readIORef q
120+ let sv =
121+ SVar { outputQueue = outQ
122+ , outputDoorBell = outQMv
123+ , readOutputQ = readOutputQBounded sv
124+ , postProcess = postProcessBounded sv
125+ , workerThreads = running
126+ , workLoop = workLoopLIFO runStreamLIFO sv q
127+ , enqueue = enqueueLIFO sv q
128+ , isWorkDone = checkEmpty
129+ , needDoorBell = wfw
130+ , svarStyle = AsyncVar
131+ , workerCount = active
132+ , accountThread = delThread sv
133+#ifdef DIAGNOSTICS
134+ , aheadWorkQueue = undefined
135+ , outputHeap = undefined
136+ , maxWorkers = maxWrk
137+ , totalDispatches = disp
138+ , maxOutQSize = maxOq
139+ , maxHeapSize = maxHs
140+ , maxWorkQSize = maxWq
141+#endif
142+ }
143+ in return sv
144+
145+getFifoSVar :: MonadAsync m => IO (SVar Stream m a)
146+getFifoSVar = do
147+ outQ <- newIORef ([], 0)
148+ outQMv <- newEmptyMVar
149+ active <- newIORef 0
150+ wfw <- newIORef False
151+ running <- newIORef S.empty
152+ q <- newQ
153+#ifdef DIAGNOSTICS
154+ disp <- newIORef 0
155+ maxWrk <- newIORef 0
156+ maxOq <- newIORef 0
157+ maxHs <- newIORef 0
158+ maxWq <- newIORef 0
159+#endif
160+ let sv =
161+ SVar { outputQueue = outQ
162+ , outputDoorBell = outQMv
163+ , readOutputQ = readOutputQBounded sv
164+ , postProcess = postProcessBounded sv
165+ , workerThreads = running
166+ , workLoop = workLoopFIFO runStreamFIFO sv q
167+ , enqueue = enqueueFIFO sv q
168+ , isWorkDone = nullQ q
169+ , needDoorBell = wfw
170+ , svarStyle = WAsyncVar
171+ , workerCount = active
172+ , accountThread = delThread sv
173+#ifdef DIAGNOSTICS
174+ , aheadWorkQueue = undefined
175+ , outputHeap = undefined
176+ , totalDispatches = disp
177+ , maxWorkers = maxWrk
178+ , maxOutQSize = maxOq
179+ , maxHeapSize = maxHs
180+ , maxWorkQSize = maxWq
181+#endif
182+ }
183+ in return sv
184+
185+{-# INLINABLE newAsyncVar #-}
186+newAsyncVar :: MonadAsync m => Stream m a -> m (SVar Stream m a)
187+newAsyncVar m = do
188+ sv <- liftIO getLifoSVar
189+ sendWorker sv m
190+
191+-- XXX Get rid of this?
192+-- | Make a stream asynchronous, triggers the computation and returns a stream
193+-- in the underlying monad representing the output generated by the original
194+-- computation. The returned action is exhaustible and must be drained once. If
195+-- not drained fully we may have a thread blocked forever and once exhausted it
196+-- will always return 'empty'.
197+--
198+-- @since 0.2.0
199+{-# INLINABLE mkAsync #-}
200+mkAsync :: (IsStream t, MonadAsync m) => t m a -> m (t m a)
201+mkAsync m = newAsyncVar (toStream m) >>= return . fromSVar
202+
203+-- | Create a new SVar and enqueue one stream computation on it.
204+{-# INLINABLE newWAsyncVar #-}
205+newWAsyncVar :: MonadAsync m => Stream m a -> m (SVar Stream m a)
206+newWAsyncVar m = do
207+ sv <- liftIO getFifoSVar
208+ sendWorker sv m
209+
210+------------------------------------------------------------------------------
211+-- Running streams concurrently
212+------------------------------------------------------------------------------
213+
214+-- Concurrency rate control.
215+--
216+-- Our objective is to create more threads on demand if the consumer is running
217+-- faster than us. As soon as we encounter a concurrent composition we create a
218+-- push pull pair of threads. We use an SVar for communication between the
219+-- consumer, pulling from the SVar and the producer who is pushing to the SVar.
220+-- The producer creates more threads if the SVar drains and becomes empty, that
221+-- is the consumer is running faster.
222+--
223+-- XXX Note 1: This mechanism can be problematic if the initial production
224+-- latency is high, we may end up creating too many threads. So we need some
225+-- way to monitor and use the latency as well. Having a limit on the dispatches
226+-- (programmer controlled) may also help.
227+--
228+-- TBD Note 2: We may want to run computations at the lower level of the
229+-- composition tree serially even when they are composed using a parallel
230+-- combinator. We can use 'serial' in place of 'async' and 'wSerial' in
231+-- place of 'wAsync'. If we find that an SVar immediately above a computation
232+-- gets drained empty we can switch to parallelizing the computation. For that
233+-- we can use a state flag to fork the rest of the computation at any point of
234+-- time inside the Monad bind operation if the consumer is running at a faster
235+-- speed.
236+--
237+-- TBD Note 3: the binary operation ('parallel') composition allows us to
238+-- dispatch a chunkSize of only 1. If we have to dispatch in arbitrary
239+-- chunksizes we will need to compose the parallel actions using a data
240+-- constructor (A Free container) instead so that we can divide it in chunks of
241+-- arbitrary size before dispatching. If the stream is composed of
242+-- hierarchically composed grains of different sizes then we can always switch
243+-- to a desired granularity depending on the consumer speed.
244+--
245+-- TBD Note 4: for pure work (when we are not in the IO monad) we can divide it
246+-- into just the number of CPUs.
247+
248+-- | Join two computations on the currently running 'SVar' queue for concurrent
249+-- execution. When we are using parallel composition, an SVar is passed around
250+-- as a state variable. We try to schedule a new parallel computation on the
251+-- SVar passed to us. The first time, when no SVar exists, a new SVar is
252+-- created. Subsequently, 'joinStreamVarAsync' may get called when a computation
253+-- already scheduled on the SVar is further evaluated. For example, when (a
254+-- `parallel` b) is evaluated it calls a 'joinStreamVarAsync' to put 'a' and 'b' on
255+-- the current scheduler queue.
256+--
257+-- The 'SVarStyle' required by the current composition context is passed as one
258+-- of the parameters. If the scheduling and composition style of the new
259+-- computation being scheduled is different than the style of the current SVar,
260+-- then we create a new SVar and schedule it on that. The newly created SVar
261+-- joins as one of the computations on the current SVar queue.
262+--
263+-- Cases when we need to switch to a new SVar:
264+--
265+-- * (x `parallel` y) `parallel` (t `parallel` u) -- all of them get scheduled on the same SVar
266+-- * (x `parallel` y) `parallel` (t `async` u) -- @t@ and @u@ get scheduled on a new child SVar
267+-- because of the scheduling policy change.
268+-- * if we 'adapt' a stream of type 'async' to a stream of type
269+-- 'Parallel', we create a new SVar at the transitioning bind.
270+-- * When the stream is switching from disjunctive composition to conjunctive
271+-- composition and vice-versa we create a new SVar to isolate the scheduling
272+-- of the two.
273+
274+forkSVarAsync :: MonadAsync m
275+ => SVarStyle -> Stream m a -> Stream m a -> Stream m a
276+forkSVarAsync style m1 m2 = Stream $ \_ stp sng yld -> do
277+ sv <- case style of
278+ AsyncVar -> newAsyncVar (concurrently m1 m2)
279+ WAsyncVar -> newWAsyncVar (concurrently m1 m2)
280+ _ -> error "illegal svar type"
281+ runStream (fromSVar sv) Nothing stp sng yld
282+ where
283+ concurrently ma mb = Stream $ \svr stp sng yld -> do
284+ liftIO $ enqueue (fromJust svr) mb
285+ (runStream ma) svr stp sng yld
286+
287+{-# INLINE joinStreamVarAsync #-}
288+joinStreamVarAsync :: MonadAsync m
289+ => SVarStyle -> Stream m a -> Stream m a -> Stream m a
290+joinStreamVarAsync style m1 m2 = Stream $ \svr stp sng yld ->
291+ case svr of
292+ Just sv | svarStyle sv == style ->
293+ liftIO (enqueue sv m2) >> (runStream m1) svr stp sng yld
294+ _ -> runStream (forkSVarAsync style m1 m2) Nothing stp sng yld
295+
296+------------------------------------------------------------------------------
297+-- Semigroup and Monoid style compositions for parallel actions
298+------------------------------------------------------------------------------
299+
300+{-# INLINE asyncS #-}
301+asyncS :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
302+asyncS = joinStreamVarAsync AsyncVar
303+
304+-- | Polymorphic version of the 'Semigroup' operation '<>' of 'AsyncT'.
305+-- Merges two streams possibly concurrently, preferring the
306+-- elements from the left one when available.
307+--
308+-- @since 0.2.0
309+{-# INLINE async #-}
310+async :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
311+async m1 m2 = fromStream $
312+ joinStreamVarAsync AsyncVar (toStream m1) (toStream m2)
313+
314+-- | Same as 'async'.
315+--
316+-- @since 0.1.0
317+{-# DEPRECATED (<|) "Please use 'async' instead." #-}
318+{-# INLINE (<|) #-}
319+(<|) :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
320+(<|) = async
321+
322+-- | XXX we can implement it more efficienty by directly implementing instead
323+-- of combining streams using async.
324+{-# INLINE consMAsync #-}
325+consMAsync :: MonadAsync m => m a -> Stream m a -> Stream m a
326+consMAsync m r = once m `asyncS` r
327+
328+------------------------------------------------------------------------------
329+-- AsyncT
330+------------------------------------------------------------------------------
331+
332+-- | Deep async composition or async composition with depth first traversal. In
333+-- a left to right 'Semigroup' composition it tries to yield elements from the
334+-- left stream as long as it can, but it can run the right stream in parallel
335+-- if it needs to, based on demand. The right stream can be run if the left
336+-- stream blocks on IO or cannot produce elements fast enough for the consumer.
337+--
338+-- @
339+-- main = ('toList' . 'asyncly' $ (fromFoldable [1,2]) \<> (fromFoldable [3,4])) >>= print
340+-- @
341+-- @
342+-- [1,2,3,4]
343+-- @
344+--
345+-- Any exceptions generated by a constituent stream are propagated to the
346+-- output stream. The output and exceptions from a single stream are guaranteed
347+-- to arrive in the same order in the resulting stream as they were generated
348+-- in the input stream. However, the relative ordering of elements from
349+-- different streams in the resulting stream can vary depending on scheduling
350+-- and generation delays.
351+--
352+-- Similarly, the monad instance of 'AsyncT' /may/ run each iteration
353+-- concurrently based on demand. More concurrent iterations are started only
354+-- if the previous iterations are not able to produce enough output for the
355+-- consumer.
356+--
357+-- @
358+-- import "Streamly"
359+-- import qualified "Streamly.Prelude" as S
360+-- import Control.Concurrent
361+--
362+-- main = 'runStream' . 'asyncly' $ do
363+-- n <- return 3 \<\> return 2 \<\> return 1
364+-- S.once $ do
365+-- threadDelay (n * 1000000)
366+-- myThreadId >>= \\tid -> putStrLn (show tid ++ ": Delay " ++ show n)
367+-- @
368+-- @
369+-- ThreadId 40: Delay 1
370+-- ThreadId 39: Delay 2
371+-- ThreadId 38: Delay 3
372+-- @
373+--
374+-- All iterations may run in the same thread if they do not block.
375+--
376+-- Note that async composition with depth first traversal can be used to
377+-- combine infinite number of streams as it explores only a bounded number of
378+-- streams at a time.
379+--
380+-- @since 0.1.0
381+newtype AsyncT m a = AsyncT {getAsyncT :: Stream m a}
382+ deriving (Functor, MonadTrans)
383+
384+-- | A demand driven left biased parallely composing IO stream of elements of
385+-- type @a@. See 'AsyncT' documentation for more details.
386+--
387+-- @since 0.2.0
388+type Async a = AsyncT IO a
389+
390+-- | Fix the type of a polymorphic stream as 'AsyncT'.
391+--
392+-- @since 0.1.0
393+asyncly :: IsStream t => AsyncT m a -> t m a
394+asyncly = adapt
395+
396+instance IsStream AsyncT where
397+ toStream = getAsyncT
398+ fromStream = AsyncT
399+
400+ {-# INLINE consM #-}
401+ {-# SPECIALIZE consM :: IO a -> AsyncT IO a -> AsyncT IO a #-}
402+ consM m r = fromStream $ consMAsync m (toStream r)
403+
404+ {-# INLINE (|:) #-}
405+ {-# SPECIALIZE (|:) :: IO a -> AsyncT IO a -> AsyncT IO a #-}
406+ (|:) = consM
407+
408+------------------------------------------------------------------------------
409+-- Semigroup
410+------------------------------------------------------------------------------
411+
412+instance MonadAsync m => Semigroup (AsyncT m a) where
413+ (<>) = async
414+
415+------------------------------------------------------------------------------
416+-- Monoid
417+------------------------------------------------------------------------------
418+
419+instance MonadAsync m => Monoid (AsyncT m a) where
420+ mempty = nil
421+ mappend = (<>)
422+
423+------------------------------------------------------------------------------
424+-- Monad
425+------------------------------------------------------------------------------
426+
427+instance MonadAsync m => Monad (AsyncT m) where
428+ return = pure
429+ (AsyncT m) >>= f = AsyncT $ bindWith asyncS m (getAsyncT . f)
430+
431+------------------------------------------------------------------------------
432+-- Other instances
433+------------------------------------------------------------------------------
434+
435+MONAD_APPLICATIVE_INSTANCE(AsyncT,MONADPARALLEL)
436+MONAD_COMMON_INSTANCES(AsyncT, MONADPARALLEL)
437+
438+------------------------------------------------------------------------------
439+-- WAsyncT
440+------------------------------------------------------------------------------
441+
442+{-# INLINE wAsyncS #-}
443+wAsyncS :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
444+wAsyncS = joinStreamVarAsync WAsyncVar
445+
446+-- | XXX we can implement it more efficienty by directly implementing instead
447+-- of combining streams using wAsync.
448+{-# INLINE consMWAsync #-}
449+consMWAsync :: MonadAsync m => m a -> Stream m a -> Stream m a
450+consMWAsync m r = once m `wAsyncS` r
451+
452+-- | Polymorphic version of the 'Semigroup' operation '<>' of 'WAsyncT'.
453+-- Merges two streams concurrently choosing elements from both fairly.
454+--
455+-- @since 0.2.0
456+{-# INLINE wAsync #-}
457+wAsync :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
458+wAsync m1 m2 = fromStream $ wAsyncS (toStream m1) (toStream m2)
459+
460+-- | Wide async composition or async composition with breadth first traversal.
461+-- The Semigroup instance of 'WAsyncT' concurrently /traverses/ the composed
462+-- streams using a depth first travesal or in a round robin fashion, yielding
463+-- elements from both streams alternately.
464+--
465+-- @
466+-- main = ('toList' . 'wAsyncly' $ (fromFoldable [1,2]) \<> (fromFoldable [3,4])) >>= print
467+-- @
468+-- @
469+-- [1,3,2,4]
470+-- @
471+--
472+-- Any exceptions generated by a constituent stream are propagated to the
473+-- output stream. The output and exceptions from a single stream are guaranteed
474+-- to arrive in the same order in the resulting stream as they were generated
475+-- in the input stream. However, the relative ordering of elements from
476+-- different streams in the resulting stream can vary depending on scheduling
477+-- and generation delays.
478+--
479+-- Similarly, the 'Monad' instance of 'WAsyncT' runs /all/ iterations fairly
480+-- concurrently using a round robin scheduling.
481+--
482+-- @
483+-- import "Streamly"
484+-- import qualified "Streamly.Prelude" as S
485+-- import Control.Concurrent
486+--
487+-- main = 'runStream' . 'wAsyncly' $ do
488+-- n <- return 3 \<\> return 2 \<\> return 1
489+-- S.once $ do
490+-- threadDelay (n * 1000000)
491+-- myThreadId >>= \\tid -> putStrLn (show tid ++ ": Delay " ++ show n)
492+-- @
493+-- @
494+-- ThreadId 40: Delay 1
495+-- ThreadId 39: Delay 2
496+-- ThreadId 38: Delay 3
497+-- @
498+--
499+-- Unlike 'AsyncT' all iterations are guaranteed to run fairly
500+-- concurrently, unconditionally.
501+--
502+-- Note that async composition with breadth first traversal can only combine a
503+-- finite number of streams as it needs to retain state for each unfinished
504+-- stream.
505+--
506+-- @since 0.2.0
507+newtype WAsyncT m a = WAsyncT {getWAsyncT :: Stream m a}
508+ deriving (Functor, MonadTrans)
509+
510+-- | A round robin parallely composing IO stream of elements of type @a@.
511+-- See 'WAsyncT' documentation for more details.
512+--
513+-- @since 0.2.0
514+type WAsync a = WAsyncT IO a
515+
516+-- | Fix the type of a polymorphic stream as 'WAsyncT'.
517+--
518+-- @since 0.2.0
519+wAsyncly :: IsStream t => WAsyncT m a -> t m a
520+wAsyncly = adapt
521+
522+instance IsStream WAsyncT where
523+ toStream = getWAsyncT
524+ fromStream = WAsyncT
525+
526+ {-# INLINE consM #-}
527+ {-# SPECIALIZE consM :: IO a -> WAsyncT IO a -> WAsyncT IO a #-}
528+ consM m r = fromStream $ consMWAsync m (toStream r)
529+
530+ {-# INLINE (|:) #-}
531+ {-# SPECIALIZE (|:) :: IO a -> WAsyncT IO a -> WAsyncT IO a #-}
532+ (|:) = consM
533+
534+------------------------------------------------------------------------------
535+-- Semigroup
536+------------------------------------------------------------------------------
537+
538+instance MonadAsync m => Semigroup (WAsyncT m a) where
539+ (<>) = wAsync
540+
541+------------------------------------------------------------------------------
542+-- Monoid
543+------------------------------------------------------------------------------
544+
545+instance MonadAsync m => Monoid (WAsyncT m a) where
546+ mempty = nil
547+ mappend = (<>)
548+
549+------------------------------------------------------------------------------
550+-- Monad
551+------------------------------------------------------------------------------
552+
553+instance MonadAsync m => Monad (WAsyncT m) where
554+ return = pure
555+ (WAsyncT m) >>= f =
556+ WAsyncT $ bindWith wAsyncS m (getWAsyncT . f)
557+
558+------------------------------------------------------------------------------
559+-- Other instances
560+------------------------------------------------------------------------------
561+
562+MONAD_APPLICATIVE_INSTANCE(WAsyncT,MONADPARALLEL)
563+MONAD_COMMON_INSTANCES(WAsyncT, MONADPARALLEL)
564+
565+-- | Same as @runStream . asyncly@.
566+--
567+-- @since 0.1.0
568+{-# DEPRECATED runAsyncT "Please use 'runStream . asyncly' instead." #-}
569+runAsyncT :: Monad m => AsyncT m a -> m ()
570+runAsyncT = run
src/Streamly/Streams/CPS.hsnew file+363
@@ -0,0 +1,363 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE ConstraintKinds #-}
3+{-# LANGUAGE FlexibleContexts #-}
4+{-# LANGUAGE FlexibleInstances #-}
5+{-# LANGUAGE InstanceSigs #-}
6+{-# LANGUAGE LambdaCase #-}
7+{-# LANGUAGE MagicHash #-}
8+{-# LANGUAGE MultiParamTypeClasses #-}
9+{-# LANGUAGE RankNTypes #-}
10+{-# LANGUAGE UnboxedTuples #-}
11+{-# LANGUAGE UndecidableInstances #-} -- XXX
12+
13+-- |
14+-- Module : Streamly.Streams.CPS
15+-- Copyright : (c) 2017 Harendra Kumar
16+--
17+-- License : BSD3
18+-- Maintainer : harendra.kumar@gmail.com
19+-- Stability : experimental
20+-- Portability : GHC
21+--
22+--
23+module Streamly.Streams.CPS
24+ (
25+ -- * Streams
26+ IsStream (..)
27+ , adapt
28+ , Streaming -- deprecated
29+ , Stream (..)
30+ , run
31+
32+ -- * Construction
33+ , nil
34+ , cons
35+ , (.:)
36+ , consMSerial
37+
38+ -- * Generation
39+ , singleton
40+ , once
41+ , repeat
42+
43+ -- * Semigroup Style Composition
44+ , serial
45+
46+ -- * Utilities
47+ , bindWith
48+ , withLocal
49+ )
50+where
51+
52+import Control.Monad.Reader.Class (MonadReader(..))
53+import Control.Monad.Trans.Class (MonadTrans(lift))
54+import Data.Semigroup (Semigroup(..))
55+import Prelude hiding (repeat)
56+
57+import Streamly.SVar
58+
59+------------------------------------------------------------------------------
60+-- The basic stream type
61+------------------------------------------------------------------------------
62+
63+-- | The type 'Stream m a' represents a monadic stream of values of type 'a'
64+-- constructed using actions in monad 'm'. It uses stop, singleton and yield
65+-- continuations equivalent to the following direct style type:
66+--
67+-- data Stream m a = Stop | Singleton a | Yield a (Stream m a)
68+--
69+-- To facilitate parallel composition we maintain a local state in an SVar that
70+-- is shared across and is used for synchronization of the streams being
71+-- composed.
72+--
73+-- The singleton case can be expressed in terms of stop and yield but we have
74+-- it as a separate case to optimize composition operations for streams with
75+-- single element. We build singleton streams in the implementation of 'pure'
76+-- for Applicative and Monad, and in 'lift' for MonadTrans.
77+--
78+newtype Stream m a =
79+ Stream {
80+ runStream :: forall r.
81+ Maybe (SVar Stream m a) -- local state
82+ -> m r -- stop
83+ -> (a -> m r) -- singleton
84+ -> (a -> Stream m a -> m r) -- yield
85+ -> m r
86+ }
87+
88+------------------------------------------------------------------------------
89+-- Types that can behave as a Stream
90+------------------------------------------------------------------------------
91+
92+infixr 5 `consM`
93+infixr 5 |:
94+
95+-- | Class of types that can represent a stream of elements of some type 'a' in
96+-- some monad 'm'.
97+--
98+-- @since 0.2.0
99+class IsStream t where
100+ toStream :: t m a -> Stream m a
101+ fromStream :: Stream m a -> t m a
102+ -- | Constructs a stream by adding a monadic action at the head of an
103+ -- existing stream. For example:
104+ --
105+ -- @
106+ -- > toList $ getLine \`consM` getLine \`consM` nil
107+ -- hello
108+ -- world
109+ -- ["hello","world"]
110+ -- @
111+ --
112+ -- /Concurrent (do not use 'parallely' to construct infinite streams)/
113+ --
114+ -- @since 0.2.0
115+ consM :: MonadAsync m => m a -> t m a -> t m a
116+ -- | Operator equivalent of 'consM'. We can read it as "@parallel colon@"
117+ -- to remember that @|@ comes before ':'.
118+ --
119+ -- @
120+ -- > toList $ getLine |: getLine |: nil
121+ -- hello
122+ -- world
123+ -- ["hello","world"]
124+ -- @
125+ --
126+ -- @
127+ -- let delay = threadDelay 1000000 >> print 1
128+ -- runStream $ serially $ delay |: delay |: delay |: nil
129+ -- runStream $ parallely $ delay |: delay |: delay |: nil
130+ -- @
131+ --
132+ -- /Concurrent (do not use 'parallely' to construct infinite streams)/
133+ --
134+ -- @since 0.2.0
135+ (|:) :: MonadAsync m => m a -> t m a -> t m a
136+ -- We can define (|:) just as 'consM' but it is defined explicitly for each
137+ -- type because we want to use SPECIALIZE pragma on the definition.
138+
139+-- | Same as 'IsStream'.
140+--
141+-- @since 0.1.0
142+{-# DEPRECATED Streaming "Please use IsStream instead." #-}
143+type Streaming = IsStream
144+
145+-------------------------------------------------------------------------------
146+-- Type adapting combinators
147+-------------------------------------------------------------------------------
148+
149+-- | Adapt any specific stream type to any other specific stream type.
150+--
151+-- @since 0.1.0
152+adapt :: (IsStream t1, IsStream t2) => t1 m a -> t2 m a
153+adapt = fromStream . toStream
154+
155+------------------------------------------------------------------------------
156+-- Construction
157+------------------------------------------------------------------------------
158+
159+-- | An empty stream.
160+--
161+-- @
162+-- > toList nil
163+-- []
164+-- @
165+--
166+-- @since 0.1.0
167+nil :: IsStream t => t m a
168+nil = fromStream $ Stream $ \_ stp _ _ -> stp
169+
170+infixr 5 `cons`
171+
172+-- faster than consM because there is no bind.
173+-- | Construct a stream by adding a pure value at the head of an existing
174+-- stream. For pure values it can be faster than 'consM'. For example:
175+--
176+-- @
177+-- > toList $ 1 \`cons` 2 \`cons` 3 \`cons` nil
178+-- [1,2,3]
179+-- @
180+--
181+-- @since 0.1.0
182+cons :: IsStream t => a -> t m a -> t m a
183+cons a r = fromStream $ Stream $ \_ _ _ yld -> yld a (toStream r)
184+
185+infixr 5 .:
186+
187+-- | Operator equivalent of 'cons'.
188+--
189+-- @
190+-- > toList $ 1 .: 2 .: 3 .: nil
191+-- [1,2,3]
192+-- @
193+--
194+-- @since 0.1.1
195+(.:) :: IsStream t => a -> t m a -> t m a
196+(.:) = cons
197+
198+-- | Constructs a stream by adding a monadic action at the head of an existing
199+-- stream. For example:
200+--
201+-- @
202+-- > toList $ getLine \`consM` getLine \`consM` nil
203+-- hello
204+-- world
205+-- ["hello","world"]
206+-- @
207+--
208+-- @since 0.2.0
209+{-# INLINE consMSerial #-}
210+consMSerial :: (Monad m) => m a -> Stream m a -> Stream m a
211+consMSerial m r = Stream $ \_ _ _ yld -> m >>= \a -> yld a r
212+
213+-------------------------------------------------------------------------------
214+-- IsStream Stream
215+-------------------------------------------------------------------------------
216+
217+instance IsStream Stream where
218+ toStream = id
219+ fromStream = id
220+
221+ {-# INLINE consM #-}
222+ {-# SPECIALIZE consM :: IO a -> Stream IO a -> Stream IO a #-}
223+ consM :: Monad m => m a -> Stream m a -> Stream m a
224+ consM = consMSerial
225+
226+ {-# INLINE (|:) #-}
227+ {-# SPECIALIZE (|:) :: IO a -> Stream IO a -> Stream IO a #-}
228+ (|:) :: Monad m => m a -> Stream m a -> Stream m a
229+ (|:) = consMSerial
230+
231+run :: (Monad m, IsStream t) => t m a -> m ()
232+run m = go (toStream m)
233+ where
234+ go m1 =
235+ let stop = return ()
236+ single _ = return ()
237+ yield _ r = go (toStream r)
238+ in (runStream m1) Nothing stop single yield
239+
240+-------------------------------------------------------------------------------
241+-- Special generation
242+-------------------------------------------------------------------------------
243+
244+-- | Same as @once . return@ but may be faster because there is no bind
245+singleton :: IsStream t => a -> t m a
246+singleton a = fromStream $ Stream $ \_ _ single _ -> single a
247+
248+-- | Create a singleton stream by executing a monadic action once. Same as
249+-- @m \`consM` nil@ but more efficient.
250+--
251+-- @
252+-- > toList $ once getLine
253+-- hello
254+-- ["hello"]
255+-- @
256+--
257+-- @since 0.2.0
258+{-# INLINE once #-}
259+once :: (Monad m, IsStream t) => m a -> t m a
260+once m = fromStream $ Stream $ \_ _ single _ -> m >>= single
261+
262+repeat :: IsStream t => a -> t m a
263+repeat a = let x = cons a x in x
264+
265+------------------------------------------------------------------------------
266+-- Semigroup
267+------------------------------------------------------------------------------
268+
269+-- | Concatenates two streams sequentially i.e. the first stream is
270+-- exhausted completely before yielding any element from the second stream.
271+{-# INLINE serial #-}
272+serial :: Stream m a -> Stream m a -> Stream m a
273+serial m1 m2 = go m1
274+ where
275+ go (Stream m) = Stream $ \_ stp sng yld ->
276+ let stop = (runStream m2) Nothing stp sng yld
277+ single a = yld a m2
278+ yield a r = yld a (go r)
279+ in m Nothing stop single yield
280+
281+instance Semigroup (Stream m a) where
282+ (<>) = serial
283+
284+------------------------------------------------------------------------------
285+-- Monoid
286+------------------------------------------------------------------------------
287+
288+instance Monoid (Stream m a) where
289+ mempty = nil
290+ mappend = (<>)
291+
292+-------------------------------------------------------------------------------
293+-- Functor
294+-------------------------------------------------------------------------------
295+
296+instance Monad m => Functor (Stream m) where
297+ fmap f m = Stream $ \_ stp sng yld ->
298+ let single = sng . f
299+ yield a r = yld (f a) (fmap f r)
300+ in (runStream m) Nothing stp single yield
301+
302+-------------------------------------------------------------------------------
303+-- Bind utility
304+-------------------------------------------------------------------------------
305+
306+{-# INLINE bindWith #-}
307+bindWith
308+ :: (forall c. Stream m c -> Stream m c -> Stream m c)
309+ -> Stream m a
310+ -> (a -> Stream m b)
311+ -> Stream m b
312+bindWith par m f = go m
313+ where
314+ go (Stream g) =
315+ Stream $ \ctx stp sng yld ->
316+ let runIt x = (runStream x) ctx stp sng yld
317+ single a = runIt $ f a
318+ yield a r = runIt $ f a `par` go r
319+ in g Nothing stp single yield
320+
321+------------------------------------------------------------------------------
322+-- Alternative & MonadPlus
323+------------------------------------------------------------------------------
324+
325+_alt :: Stream m a -> Stream m a -> Stream m a
326+_alt m1 m2 = Stream $ \_ stp sng yld ->
327+ let stop = runStream m2 Nothing stp sng yld
328+ in runStream m1 Nothing stop sng yld
329+
330+------------------------------------------------------------------------------
331+-- MonadReader
332+------------------------------------------------------------------------------
333+
334+withLocal :: MonadReader r m => (r -> r) -> Stream m a -> Stream m a
335+withLocal f m =
336+ Stream $ \_ stp sng yld ->
337+ let single = local f . sng
338+ yield a r = local f $ yld a (withLocal f r)
339+ in (runStream m) Nothing (local f stp) single yield
340+
341+------------------------------------------------------------------------------
342+-- MonadError
343+------------------------------------------------------------------------------
344+
345+{-
346+-- XXX handle and test cross thread state transfer
347+withCatchError
348+ :: MonadError e m
349+ => Stream m a -> (e -> Stream m a) -> Stream m a
350+withCatchError m h =
351+ Stream $ \_ stp sng yld ->
352+ let run x = runStream x Nothing stp sng yield
353+ handle r = r `catchError` \e -> run $ h e
354+ yield a r = yld a (withCatchError r h)
355+ in handle $ run m
356+-}
357+
358+-------------------------------------------------------------------------------
359+-- Transformers
360+-------------------------------------------------------------------------------
361+
362+instance MonadTrans Stream where
363+ lift = once
src/Streamly/Streams/Instances.hsnew file+40
@@ -0,0 +1,40 @@
1+------------------------------------------------------------------------------
2+-- CPP macros for common instances
3+------------------------------------------------------------------------------
4+
5+-- XXX use template haskell instead and include Monoid and IsStream instances
6+-- as well.
7+
8+#define MONADPARALLEL , MonadAsync m
9+
10+#define MONAD_APPLICATIVE_INSTANCE(STREAM,CONSTRAINT) \
11+instance (Monad m CONSTRAINT) => Applicative (STREAM m) where { \
12+ pure = STREAM . singleton; \
13+ (<*>) = ap }
14+
15+#define MONAD_COMMON_INSTANCES(STREAM,CONSTRAINT) \
16+instance (MonadBase b m, Monad m CONSTRAINT) => MonadBase b (STREAM m) where {\
17+ liftBase = liftBaseDefault }; \
18+ \
19+instance (MonadIO m CONSTRAINT) => MonadIO (STREAM m) where { \
20+ liftIO = lift . liftIO }; \
21+ \
22+instance (MonadThrow m CONSTRAINT) => MonadThrow (STREAM m) where { \
23+ throwM = lift . throwM }; \
24+ \
25+{- \
26+instance (MonadError e m CONSTRAINT) => MonadError e (STREAM m) where { \
27+ throwError = lift . throwError; \
28+ catchError m h = \
29+ fromStream $ withCatchError (toStream m) (\e -> toStream $ h e) }; \
30+-} \
31+ \
32+instance (MonadReader r m CONSTRAINT) => MonadReader r (STREAM m) where { \
33+ ask = lift ask; \
34+ local f m = fromStream $ withLocal f (toStream m) }; \
35+ \
36+instance (MonadState s m CONSTRAINT) => MonadState s (STREAM m) where { \
37+ get = lift get; \
38+ put x = lift (put x); \
39+ state k = lift (state k) }
40+
src/Streamly/Streams/Parallel.hsnew file+372
@@ -0,0 +1,372 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE ConstraintKinds #-}
3+{-# LANGUAGE FlexibleContexts #-}
4+{-# LANGUAGE FlexibleInstances #-}
5+{-# LANGUAGE GeneralizedNewtypeDeriving#-}
6+{-# LANGUAGE InstanceSigs #-}
7+{-# LANGUAGE MultiParamTypeClasses #-}
8+{-# LANGUAGE StandaloneDeriving #-}
9+{-# LANGUAGE UndecidableInstances #-} -- XXX
10+
11+-- |
12+-- Module : Streamly.Streams.Parallel
13+-- Copyright : (c) 2017 Harendra Kumar
14+--
15+-- License : BSD3
16+-- Maintainer : harendra.kumar@gmail.com
17+-- Stability : experimental
18+-- Portability : GHC
19+--
20+--
21+module Streamly.Streams.Parallel
22+ (
23+ ParallelT
24+ , Parallel
25+ , parallely
26+ , parallel
27+ , runParallelT -- deprecated
28+
29+ -- * Function application
30+ , mkParallel
31+ , (|$)
32+ , (|&)
33+ , (|$.)
34+ , (|&.)
35+ )
36+where
37+
38+import Control.Monad (ap)
39+import Control.Monad.Base (MonadBase(..), liftBaseDefault)
40+import Control.Monad.Catch (MonadThrow, throwM)
41+-- import Control.Monad.Error.Class (MonadError(..))
42+import Control.Monad.IO.Class (MonadIO(..))
43+import Control.Monad.Reader.Class (MonadReader(..))
44+import Control.Monad.State.Class (MonadState(..))
45+import Control.Monad.Trans.Class (MonadTrans(lift))
46+import Data.Functor (void)
47+import Data.Semigroup (Semigroup(..))
48+
49+import Streamly.Streams.SVar (fromSVar)
50+import Streamly.Streams.CPS
51+import Streamly.SVar
52+
53+#include "Instances.hs"
54+
55+-------------------------------------------------------------------------------
56+-- Parallel
57+-------------------------------------------------------------------------------
58+
59+{-# NOINLINE runOne #-}
60+runOne :: MonadIO m => SVar Stream m a -> Stream m a -> m ()
61+runOne sv m = (runStream m) (Just sv) stop single yield
62+
63+ where
64+
65+ stop = liftIO $ sendStop sv
66+ sendit a = liftIO $ send sv (ChildYield a)
67+ single a = sendit a >> stop
68+ -- XXX there is no flow control in parallel case. We should perhaps use a
69+ -- queue and queue it back on that and exit the thread when the outputQueue
70+ -- overflows. Parallel is dangerous because it can accumulate unbounded
71+ -- output in the buffer.
72+ yield a r = void (sendit a) >> runOne sv r
73+
74+{-# NOINLINE forkSVarPar #-}
75+forkSVarPar :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
76+forkSVarPar m r = Stream $ \_ stp sng yld -> do
77+ sv <- newParallelVar
78+ pushWorkerPar sv (runOne sv m)
79+ pushWorkerPar sv (runOne sv r)
80+ (runStream (fromSVar sv)) Nothing stp sng yld
81+
82+{-# INLINE joinStreamVarPar #-}
83+joinStreamVarPar :: MonadAsync m
84+ => SVarStyle -> Stream m a -> Stream m a -> Stream m a
85+joinStreamVarPar style m1 m2 = Stream $ \svr stp sng yld ->
86+ case svr of
87+ Just sv | svarStyle sv == style -> do
88+ pushWorkerPar sv (runOne sv m1) >> (runStream m2) svr stp sng yld
89+ _ -> runStream (forkSVarPar m1 m2) Nothing stp sng yld
90+
91+{-# INLINE parallelStream #-}
92+parallelStream :: MonadAsync m => Stream m a -> Stream m a -> Stream m a
93+parallelStream = joinStreamVarPar ParallelVar
94+
95+-- | XXX we can implement it more efficienty by directly implementing instead
96+-- of combining streams using parallel.
97+{-# INLINE consMParallel #-}
98+consMParallel :: MonadAsync m => m a -> Stream m a -> Stream m a
99+consMParallel m r = once m `parallelStream` r
100+
101+-- | Polymorphic version of the 'Semigroup' operation '<>' of 'ParallelT'
102+-- Merges two streams concurrently.
103+--
104+-- @since 0.2.0
105+{-# INLINE parallel #-}
106+parallel :: (IsStream t, MonadAsync m) => t m a -> t m a -> t m a
107+parallel m1 m2 = fromStream $ parallelStream (toStream m1) (toStream m2)
108+
109+------------------------------------------------------------------------------
110+-- Convert a stream to parallel
111+------------------------------------------------------------------------------
112+
113+mkParallel :: (IsStream t, MonadAsync m) => t m a -> m (t m a)
114+mkParallel m = do
115+ sv <- newParallelVar
116+ pushWorkerPar sv (runOne sv $ toStream m)
117+ return $ fromSVar sv
118+
119+------------------------------------------------------------------------------
120+-- Stream to stream concurrent function application
121+------------------------------------------------------------------------------
122+
123+{-# INLINE applyWith #-}
124+applyWith :: (IsStream t, MonadAsync m) => (t m a -> t m b) -> t m a -> t m b
125+applyWith f m = fromStream $ Stream $ \svr stp sng yld -> do
126+ sv <- newParallelVar
127+ pushWorkerPar sv (runOne sv (toStream m))
128+ runStream (toStream $ f $ fromSVar sv) svr stp sng yld
129+
130+------------------------------------------------------------------------------
131+-- Stream runner concurrent function application
132+------------------------------------------------------------------------------
133+
134+{-# INLINE runWith #-}
135+runWith :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> m b
136+runWith f m = do
137+ sv <- newParallelVar
138+ pushWorkerPar sv (runOne sv $ toStream m)
139+ f $ fromSVar sv
140+
141+------------------------------------------------------------------------------
142+-- Concurrent Application
143+------------------------------------------------------------------------------
144+
145+infixr 0 |$
146+infixr 0 |$.
147+
148+infixl 1 |&
149+infixl 1 |&.
150+
151+-- | Parallel function application operator for streams; just like the regular
152+-- function application operator '$' except that it is concurrent. The
153+-- following code prints a value every second even though each stage adds a 1
154+-- second delay.
155+--
156+--
157+-- @
158+-- runStream $
159+-- S.mapM (\\x -> threadDelay 1000000 >> print x)
160+-- |$ S.repeatM (threadDelay 1000000 >> return 1)
161+-- @
162+--
163+-- /Concurrent/
164+--
165+-- @since 0.3.0
166+{-# INLINE (|$) #-}
167+(|$) :: (IsStream t, MonadAsync m) => (t m a -> t m b) -> t m a -> t m b
168+f |$ x = applyWith f x
169+
170+-- | Parallel reverse function application operator for streams; just like the
171+-- regular reverse function application operator '&' except that it is
172+-- concurrent.
173+--
174+-- @
175+-- runStream $
176+-- S.repeatM (threadDelay 1000000 >> return 1)
177+-- |& S.mapM (\\x -> threadDelay 1000000 >> print x)
178+-- @
179+--
180+-- /Concurrent/
181+--
182+-- @since 0.3.0
183+{-# INLINE (|&) #-}
184+(|&) :: (IsStream t, MonadAsync m) => t m a -> (t m a -> t m b) -> t m b
185+x |& f = f |$ x
186+
187+-- | Parallel function application operator; applies a @run@ or @fold@ function
188+-- to a stream such that the fold consumer and the stream producer run in
189+-- parallel. A @run@ or @fold@ function reduces the stream to a value in the
190+-- underlying monad. The @.@ at the end of the operator is a mnemonic for
191+-- termination of the stream.
192+--
193+-- @
194+-- S.foldlM' (\\_ a -> threadDelay 1000000 >> print a) ()
195+-- |$. S.repeatM (threadDelay 1000000 >> return 1)
196+-- @
197+--
198+-- /Concurrent/
199+--
200+-- @since 0.3.0
201+{-# INLINE (|$.) #-}
202+(|$.) :: (IsStream t, MonadAsync m) => (t m a -> m b) -> t m a -> m b
203+f |$. x = runWith f x
204+
205+-- | Parallel reverse function application operator for applying a run or fold
206+-- functions to a stream. Just like '|$.' except that the operands are reversed.
207+--
208+-- @
209+-- S.repeatM (threadDelay 1000000 >> return 1)
210+-- |&. S.foldlM' (\\_ a -> threadDelay 1000000 >> print a) ()
211+-- @
212+--
213+-- /Concurrent/
214+--
215+-- @since 0.3.0
216+{-# INLINE (|&.) #-}
217+(|&.) :: (IsStream t, MonadAsync m) => t m a -> (t m a -> m b) -> m b
218+x |&. f = f |$. x
219+
220+------------------------------------------------------------------------------
221+-- ParallelT
222+------------------------------------------------------------------------------
223+
224+-- | Async composition with simultaneous traversal of all streams.
225+--
226+-- The Semigroup instance of 'ParallelT' concurrently /merges/ two streams,
227+-- running both strictly concurrently and yielding elements from both streams
228+-- as they arrive. When multiple streams are combined using 'ParallelT' each
229+-- one is evaluated in its own thread and the results produced are presented in
230+-- the combined stream on a first come first serve basis.
231+--
232+-- 'AsyncT' and 'WAsyncT' are /concurrent lookahead streams/ each with a
233+-- specific type of consumption pattern (depth first or breadth first). Since
234+-- they are lookahead, they may introduce certain default latency in starting
235+-- more concurrent tasks for efficiency reasons or may put a default limitation
236+-- on the resource consumption (e.g. number of concurrent threads for
237+-- lookahead). If we look at the implementation detail, they both can share a
238+-- pool of worker threads to evaluate the streams in the desired pattern and at
239+-- the desired rate. However, 'ParallelT' uses a separate runtime thread to
240+-- evaluate each stream.
241+--
242+-- 'WAsyncT' is similar to 'ParallelT', as both of them evaluate the
243+-- constituent streams fairly in a round robin fashion.
244+-- However, the key difference is that 'WAsyncT' is lazy or pull driven
245+-- whereas 'ParallelT' is strict or push driven. 'ParallelT' immediately
246+-- starts concurrent evaluation of both the streams (in separate threads) and
247+-- later picks the results whereas 'WAsyncT' may wait for a certain latency
248+-- threshold before initiating concurrent evaluation of the next stream. The
249+-- concurrent scheduling of the next stream or the degree of concurrency is
250+-- driven by the feedback from the consumer. In case of 'ParallelT' each stream
251+-- is evaluated in a separate thread and results are /pushed/ to a shared
252+-- output buffer, the evaluation rate is controlled by blocking when the buffer
253+-- is full.
254+--
255+-- Concurrent lookahead streams are generally more efficient than
256+-- 'ParallelT' and can work pretty efficiently even for smaller tasks because
257+-- they do not necessarily use a separate thread for each task. So they should
258+-- be preferred over 'ParallelT' especially when efficiency is a concern and
259+-- simultaneous strict evaluation is not a requirement. 'ParallelT' is useful
260+-- for cases when the streams are required to be evaluated simultaneously
261+-- irrespective of how the consumer consumes them e.g. when we want to race
262+-- two tasks and want to start both strictly at the same time or if we have
263+-- timers in the parallel tasks and our results depend on the timers being
264+-- started at the same time. We can say that 'ParallelT' is almost the same
265+-- (modulo some implementation differences) as 'WAsyncT' when the latter is
266+-- used with unlimited lookahead and zero latency in initiating lookahead.
267+--
268+-- @
269+-- main = ('toList' . 'parallely' $ (fromFoldable [1,2]) \<> (fromFoldable [3,4])) >>= print
270+-- @
271+-- @
272+-- [1,3,2,4]
273+-- @
274+--
275+-- When streams with more than one element are merged, it yields whichever
276+-- stream yields first without any bias, unlike the 'Async' style streams.
277+--
278+-- Any exceptions generated by a constituent stream are propagated to the
279+-- output stream. The output and exceptions from a single stream are guaranteed
280+-- to arrive in the same order in the resulting stream as they were generated
281+-- in the input stream. However, the relative ordering of elements from
282+-- different streams in the resulting stream can vary depending on scheduling
283+-- and generation delays.
284+--
285+-- Similarly, the 'Monad' instance of 'ParallelT' runs /all/ iterations
286+-- of the loop concurrently.
287+--
288+-- @
289+-- import "Streamly"
290+-- import qualified "Streamly.Prelude" as S
291+-- import Control.Concurrent
292+--
293+-- main = 'runStream' . 'parallely' $ do
294+-- n <- return 3 \<\> return 2 \<\> return 1
295+-- S.once $ do
296+-- threadDelay (n * 1000000)
297+-- myThreadId >>= \\tid -> putStrLn (show tid ++ ": Delay " ++ show n)
298+-- @
299+-- @
300+-- ThreadId 40: Delay 1
301+-- ThreadId 39: Delay 2
302+-- ThreadId 38: Delay 3
303+-- @
304+--
305+-- Note that parallel composition can only combine a finite number of
306+-- streams as it needs to retain state for each unfinished stream.
307+--
308+-- @since 0.1.0
309+newtype ParallelT m a = ParallelT {getParallelT :: Stream m a}
310+ deriving (Functor, MonadTrans)
311+
312+-- | A parallely composing IO stream of elements of type @a@.
313+-- See 'ParallelT' documentation for more details.
314+--
315+-- @since 0.2.0
316+type Parallel a = ParallelT IO a
317+
318+-- | Fix the type of a polymorphic stream as 'ParallelT'.
319+--
320+-- @since 0.1.0
321+parallely :: IsStream t => ParallelT m a -> t m a
322+parallely = adapt
323+
324+instance IsStream ParallelT where
325+ toStream = getParallelT
326+ fromStream = ParallelT
327+
328+ {-# INLINE consM #-}
329+ {-# SPECIALIZE consM :: IO a -> ParallelT IO a -> ParallelT IO a #-}
330+ consM m r = fromStream $ consMParallel m (toStream r)
331+
332+ {-# INLINE (|:) #-}
333+ {-# SPECIALIZE (|:) :: IO a -> ParallelT IO a -> ParallelT IO a #-}
334+ (|:) = consM
335+
336+------------------------------------------------------------------------------
337+-- Semigroup
338+------------------------------------------------------------------------------
339+
340+instance MonadAsync m => Semigroup (ParallelT m a) where
341+ (<>) = parallel
342+
343+------------------------------------------------------------------------------
344+-- Monoid
345+------------------------------------------------------------------------------
346+
347+instance MonadAsync m => Monoid (ParallelT m a) where
348+ mempty = nil
349+ mappend = (<>)
350+
351+------------------------------------------------------------------------------
352+-- Monad
353+------------------------------------------------------------------------------
354+
355+instance MonadAsync m => Monad (ParallelT m) where
356+ return = pure
357+ (ParallelT m) >>= f
358+ = ParallelT $ bindWith parallelStream m (getParallelT . f)
359+
360+------------------------------------------------------------------------------
361+-- Other instances
362+------------------------------------------------------------------------------
363+
364+MONAD_APPLICATIVE_INSTANCE(ParallelT,MONADPARALLEL)
365+MONAD_COMMON_INSTANCES(ParallelT, MONADPARALLEL)
366+
367+-- | Same as @runStream . parallely@.
368+--
369+-- @since 0.1.0
370+{-# DEPRECATED runParallelT "Please use 'runStream . parallely' instead." #-}
371+runParallelT :: Monad m => ParallelT m a -> m ()
372+runParallelT = run
src/Streamly/Streams/Prelude.hsnew file+164
@@ -0,0 +1,164 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE ConstraintKinds #-}
3+{-# LANGUAGE FlexibleContexts #-}
4+{-# LANGUAGE FlexibleInstances #-}
5+{-# LANGUAGE GeneralizedNewtypeDeriving#-}
6+{-# LANGUAGE InstanceSigs #-}
7+{-# LANGUAGE MultiParamTypeClasses #-}
8+{-# LANGUAGE RankNTypes #-}
9+{-# LANGUAGE StandaloneDeriving #-}
10+{-# LANGUAGE UndecidableInstances #-} -- XXX
11+
12+-- |
13+-- Module : Streamly.Streams.Prelude
14+-- Copyright : (c) 2017 Harendra Kumar
15+--
16+-- License : BSD3
17+-- Maintainer : harendra.kumar@gmail.com
18+-- Stability : experimental
19+-- Portability : GHC
20+--
21+--
22+module Streamly.Streams.Prelude
23+ (
24+ -- * Construction
25+ streamBuild
26+ , fromCallback
27+
28+ -- * Elimination
29+ , streamFold
30+ , runStream
31+ , runStreaming -- deprecated
32+
33+ -- * Fold Utilities
34+ , foldWith
35+ , foldMapWith
36+ , forEachWith
37+ )
38+where
39+
40+import Streamly.Streams.Serial (SerialT)
41+import Streamly.SVar (SVar)
42+
43+import Streamly.Streams.CPS hiding (runStream)
44+import qualified Streamly.Streams.CPS as C
45+
46+------------------------------------------------------------------------------
47+-- Fold Utilities
48+------------------------------------------------------------------------------
49+
50+-- | A variant of 'Data.Foldable.fold' that allows you to fold a 'Foldable'
51+-- container of streams using the specified stream sum operation.
52+--
53+-- @foldWith 'async' $ map return [1..3]@
54+--
55+-- @since 0.1.0
56+{-# INLINABLE foldWith #-}
57+foldWith :: (IsStream t, Foldable f)
58+ => (t m a -> t m a -> t m a) -> f (t m a) -> t m a
59+foldWith f = foldr f nil
60+
61+-- | A variant of 'foldMap' that allows you to map a monadic streaming action
62+-- on a 'Foldable' container and then fold it using the specified stream sum
63+-- operation.
64+--
65+-- @foldMapWith 'async' return [1..3]@
66+--
67+-- @since 0.1.0
68+{-# INLINABLE foldMapWith #-}
69+foldMapWith :: (IsStream t, Foldable f)
70+ => (t m b -> t m b -> t m b) -> (a -> t m b) -> f a -> t m b
71+foldMapWith f g = foldr (f . g) nil
72+
73+-- | Like 'foldMapWith' but with the last two arguments reversed i.e. the
74+-- monadic streaming function is the last argument.
75+--
76+-- @since 0.1.0
77+{-# INLINABLE forEachWith #-}
78+forEachWith :: (IsStream t, Foldable f)
79+ => (t m b -> t m b -> t m b) -> f a -> (a -> t m b) -> t m b
80+forEachWith f xs g = foldr (f . g) nil xs
81+
82+------------------------------------------------------------------------------
83+-- Generation
84+------------------------------------------------------------------------------
85+
86+-- | Build a stream from its church encoding. The function passed maps
87+-- directly to the underlying representation of the stream type. The second
88+-- parameter to the function is the "yield" function yielding a value and the
89+-- remaining stream if any otherwise 'Nothing'. The third parameter is to
90+-- represent an "empty" stream.
91+streamBuild :: IsStream t
92+ => (forall r. Maybe (SVar Stream m a)
93+ -> (a -> t m a -> m r)
94+ -> (a -> m r)
95+ -> m r
96+ -> m r)
97+ -> t m a
98+streamBuild k = fromStream $ Stream $ \svr stp sng yld ->
99+ let yield a r = yld a (toStream r)
100+ in k svr yield sng stp
101+
102+-- | Build a singleton stream from a callback function.
103+fromCallback :: IsStream t => (forall r. (a -> m r) -> m r) -> t m a
104+fromCallback k = fromStream $ Stream $ \_ _ sng _ -> k sng
105+
106+-------------------------------------------------------------------------------
107+-- Generation by unfold
108+-------------------------------------------------------------------------------
109+
110+{-
111+{-# INLINE unfoldr #-}
112+unfoldr :: (b -> Maybe (a, b)) -> b -> Stream m a
113+unfoldr step = go
114+ where
115+ go s = Stream $ \_ stp _ yld ->
116+ case step s of
117+ Nothing -> stp
118+ Just (a, b) -> yld a (go b)
119+
120+{-# INLINE unfoldrM #-}
121+unfoldrM :: Monad m => (b -> m (Maybe (a, b))) -> b -> Stream m a
122+unfoldrM step = go
123+ where
124+ go s = Stream $ \svr stp sng yld -> do
125+ mayb <- step s
126+ case mayb of
127+ Nothing -> stp
128+ Just (a, b) ->
129+ C.runStream (a `cons` go b) svr stp sng yld
130+-}
131+
132+------------------------------------------------------------------------------
133+-- Destroying a stream
134+------------------------------------------------------------------------------
135+
136+-- | Fold a stream using its church encoding. The second argument is the "step"
137+-- function consuming an element and the remaining stream, if any. The third
138+-- argument is for consuming an "empty" stream that yields nothing.
139+streamFold
140+ :: IsStream t
141+ => Maybe (SVar Stream m a)
142+ -> (a -> t m a -> m r)
143+ -> (a -> m r)
144+ -> m r
145+ -> t m a
146+ -> m r
147+streamFold svr step single blank m =
148+ let yield a x = step a (fromStream x)
149+ in (C.runStream (toStream m)) svr blank single yield
150+
151+-- | Run a streaming composition, discard the results. By default it interprets
152+-- the stream as 'SerialT', to run other types of streams use the type adapting
153+-- combinators for example @runStream . 'asyncly'@.
154+--
155+-- @since 0.2.0
156+runStream :: Monad m => SerialT m a -> m ()
157+runStream = run
158+
159+-- | Same as 'runStream'
160+--
161+-- @since 0.1.0
162+{-# DEPRECATED runStreaming "Please use runStream instead." #-}
163+runStreaming :: (Monad m, IsStream t) => t m a -> m ()
164+runStreaming = runStream . adapt
src/Streamly/Streams/SVar.hsnew file+84
@@ -0,0 +1,84 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE ConstraintKinds #-}
3+{-# LANGUAGE FlexibleContexts #-}
4+{-# LANGUAGE FlexibleInstances #-}
5+{-# LANGUAGE LambdaCase #-}
6+{-# LANGUAGE MagicHash #-}
7+{-# LANGUAGE MultiParamTypeClasses #-}
8+{-# LANGUAGE RankNTypes #-}
9+{-# LANGUAGE UnboxedTuples #-}
10+{-# LANGUAGE UndecidableInstances #-} -- XXX
11+
12+-- |
13+-- Module : Streamly.CPS.Concurrent
14+-- Copyright : (c) 2017 Harendra Kumar
15+--
16+-- License : BSD3
17+-- Maintainer : harendra.kumar@gmail.com
18+-- Stability : experimental
19+-- Portability : GHC
20+--
21+--
22+module Streamly.Streams.SVar
23+ (
24+ fromSVar
25+ , toSVar
26+ )
27+where
28+
29+import Control.Monad.Catch (throwM)
30+
31+import Streamly.SVar
32+import Streamly.Streams.CPS
33+
34+-- MVar diagnostics has some overhead - around 5% on asyncly null benchmark, we
35+-- can keep it on in production to debug problems quickly if and when they
36+-- happen, but it may result in unexpected output when threads are left hanging
37+-- until they are GCed because the consumer went away.
38+
39+-- | Pull a stream from an SVar.
40+{-# NOINLINE fromStreamVar #-}
41+fromStreamVar :: MonadAsync m => SVar Stream m a -> Stream m a
42+fromStreamVar sv = Stream $ \_ stp sng yld -> do
43+ list <- readOutputQ sv
44+ -- Reversing the output is important to guarantee that we process the
45+ -- outputs in the same order as they were generated by the constituent
46+ -- streams.
47+ runStream (processEvents $ reverse list) Nothing stp sng yld
48+
49+ where
50+
51+ allDone stp = do
52+#ifdef DIAGNOSTICS
53+#ifdef DIAGNOSTICS_VERBOSE
54+ svInfo <- liftIO $ dumpSVar sv
55+ liftIO $ hPutStrLn stderr $ "fromStreamVar done\n" ++ svInfo
56+#endif
57+#endif
58+ stp
59+
60+ {-# INLINE processEvents #-}
61+ processEvents [] = Stream $ \_ stp sng yld -> do
62+ done <- postProcess sv
63+ if done
64+ then allDone stp
65+ else runStream (fromStreamVar sv) Nothing stp sng yld
66+
67+ processEvents (ev : es) = Stream $ \_ stp sng yld -> do
68+ let rest = processEvents es
69+ case ev of
70+ ChildYield a -> yld a rest
71+ ChildStop tid e -> do
72+ accountThread sv tid
73+ case e of
74+ Nothing -> runStream rest Nothing stp sng yld
75+ Just ex -> throwM ex
76+
77+{-# INLINE fromSVar #-}
78+fromSVar :: (MonadAsync m, IsStream t) => SVar Stream m a -> t m a
79+fromSVar sv = fromStream $ fromStreamVar sv
80+
81+-- | Write a stream to an 'SVar' in a non-blocking manner. The stream can then
82+-- be read back from the SVar using 'fromSVar'.
83+toSVar :: (IsStream t, MonadAsync m) => SVar Stream m a -> t m a -> m ()
84+toSVar sv m = toStreamVar sv (toStream m)
src/Streamly/Streams/Serial.hsnew file+344
@@ -0,0 +1,344 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE ConstraintKinds #-}
3+{-# LANGUAGE FlexibleContexts #-}
4+{-# LANGUAGE FlexibleInstances #-}
5+{-# LANGUAGE GeneralizedNewtypeDeriving#-}
6+{-# LANGUAGE InstanceSigs #-}
7+{-# LANGUAGE MultiParamTypeClasses #-}
8+{-# LANGUAGE StandaloneDeriving #-}
9+{-# LANGUAGE UndecidableInstances #-} -- XXX
10+
11+-- |
12+-- Module : Streamly.Streams.Serial
13+-- Copyright : (c) 2017 Harendra Kumar
14+--
15+-- License : BSD3
16+-- Maintainer : harendra.kumar@gmail.com
17+-- Stability : experimental
18+-- Portability : GHC
19+--
20+--
21+module Streamly.Streams.Serial
22+ (
23+ -- * Serial appending stream
24+ SerialT
25+ , StreamT -- deprecated
26+ , Serial
27+ , serial
28+ , serially
29+
30+ -- * Serial interleaving stream
31+ , WSerialT
32+ , InterleavedT -- deprecated
33+ , WSerial
34+ , wSerial
35+ , (<=>) -- deprecated
36+ , wSerially
37+ , interleaving -- deprecated
38+
39+ -- * Running Streams
40+ , runStreamT -- deprecated
41+ , runInterleavedT -- deprecated
42+ )
43+where
44+
45+import Control.Monad (ap)
46+import Control.Monad.Base (MonadBase(..), liftBaseDefault)
47+import Control.Monad.Catch (MonadThrow, throwM)
48+-- import Control.Monad.Error.Class (MonadError(..))
49+import Control.Monad.IO.Class (MonadIO(..))
50+import Control.Monad.Reader.Class (MonadReader(..))
51+import Control.Monad.State.Class (MonadState(..))
52+import Control.Monad.Trans.Class (MonadTrans(lift))
53+import Data.Semigroup (Semigroup(..))
54+
55+import Streamly.Streams.CPS hiding (serial)
56+import qualified Streamly.Streams.CPS as C
57+
58+#include "Instances.hs"
59+
60+------------------------------------------------------------------------------
61+-- SerialT
62+------------------------------------------------------------------------------
63+
64+-- | Deep serial composition or serial composition with depth first traversal.
65+-- The 'Semigroup' instance of 'SerialT' appends two streams serially in a
66+-- depth first manner, yielding all elements from the first stream, and then
67+-- all elements from the second stream.
68+--
69+-- @
70+-- import Streamly
71+-- import qualified "Streamly.Prelude" as S
72+--
73+-- main = ('toList' . 'serially' $ (fromFoldable [1,2]) \<\> (fromFoldable [3,4])) >>= print
74+-- @
75+-- @
76+-- [1,2,3,4]
77+-- @
78+--
79+-- The 'Monad' instance runs the /monadic continuation/ for each
80+-- element of the stream, serially.
81+--
82+-- @
83+-- main = 'runStream' . 'serially' $ do
84+-- x <- return 1 \<\> return 2
85+-- S.once $ print x
86+-- @
87+-- @
88+-- 1
89+-- 2
90+-- @
91+--
92+-- 'SerialT' nests streams serially in a depth first manner.
93+--
94+-- @
95+-- main = 'runStream' . 'serially' $ do
96+-- x <- return 1 \<\> return 2
97+-- y <- return 3 \<\> return 4
98+-- S.once $ print (x, y)
99+-- @
100+-- @
101+-- (1,3)
102+-- (1,4)
103+-- (2,3)
104+-- (2,4)
105+-- @
106+--
107+-- This behavior of 'SerialT' is exactly like a list transformer. We call the
108+-- monadic code being run for each element of the stream a monadic
109+-- continuation. In imperative paradigm we can think of this composition as
110+-- nested @for@ loops and the monadic continuation is the body of the loop. The
111+-- loop iterates for all elements of the stream.
112+--
113+-- The 'serially' combinator can be omitted as the default stream type is
114+-- 'SerialT'.
115+-- Note that serial composition with depth first traversal can be used to
116+-- combine an infinite number of streams as it explores only one stream at a
117+-- time.
118+--
119+-- @since 0.2.0
120+newtype SerialT m a = SerialT {getSerialT :: Stream m a}
121+ deriving (Semigroup, Monoid, Functor, MonadTrans)
122+
123+-- | A serial IO stream of elements of type @a@. See 'SerialT' documentation
124+-- for more details.
125+--
126+-- @since 0.2.0
127+type Serial a = SerialT IO a
128+
129+-- |
130+-- @since 0.1.0
131+{-# DEPRECATED StreamT "Please use 'SerialT' instead." #-}
132+type StreamT = SerialT
133+
134+-- | Fix the type of a polymorphic stream as 'SerialT'.
135+--
136+-- @since 0.1.0
137+serially :: IsStream t => SerialT m a -> t m a
138+serially = adapt
139+
140+instance IsStream SerialT where
141+ toStream = getSerialT
142+ fromStream = SerialT
143+
144+ {-# INLINE consM #-}
145+ {-# SPECIALIZE consM :: IO a -> SerialT IO a -> SerialT IO a #-}
146+ consM :: Monad m => m a -> SerialT m a -> SerialT m a
147+ consM m r = fromStream $ consMSerial m (toStream r)
148+
149+ {-# INLINE (|:) #-}
150+ {-# SPECIALIZE (|:) :: IO a -> SerialT IO a -> SerialT IO a #-}
151+ (|:) :: Monad m => m a -> SerialT m a -> SerialT m a
152+ m |: r = fromStream $ consMSerial m (toStream r)
153+
154+------------------------------------------------------------------------------
155+-- Semigroup
156+------------------------------------------------------------------------------
157+
158+-- | Polymorphic version of the 'Semigroup' operation '<>' of 'SerialT'.
159+-- Appends two streams sequentially, yielding all elements from the first
160+-- stream, and then all elements from the second stream.
161+--
162+-- @since 0.2.0
163+{-# INLINE serial #-}
164+serial :: IsStream t => t m a -> t m a -> t m a
165+serial m1 m2 = fromStream $ C.serial (toStream m1) (toStream m2)
166+
167+------------------------------------------------------------------------------
168+-- Monad
169+------------------------------------------------------------------------------
170+
171+instance Monad m => Monad (SerialT m) where
172+ return = pure
173+ (SerialT (Stream m)) >>= f = SerialT $ Stream $ \_ stp sng yld ->
174+ let runIt x = (runStream x) Nothing stp sng yld
175+ single a = runIt $ toStream (f a)
176+ yield a r = runIt $ toStream $ f a <> (fromStream r >>= f)
177+ in m Nothing stp single yield
178+
179+------------------------------------------------------------------------------
180+-- Other instances
181+------------------------------------------------------------------------------
182+
183+MONAD_APPLICATIVE_INSTANCE(SerialT,)
184+MONAD_COMMON_INSTANCES(SerialT,)
185+
186+------------------------------------------------------------------------------
187+-- WSerialT
188+------------------------------------------------------------------------------
189+
190+-- | Wide serial composition or serial composition with a breadth first
191+-- traversal. The 'Semigroup' instance of 'WSerialT' traverses
192+-- the two streams in a breadth first manner. In other words, it interleaves
193+-- two streams, yielding one element from each stream alternately.
194+--
195+-- @
196+-- import Streamly
197+-- import qualified "Streamly.Prelude" as S
198+--
199+-- main = ('toList' . 'wSerially' $ (fromFoldable [1,2]) \<\> (fromFoldable [3,4])) >>= print
200+-- @
201+-- @
202+-- [1,3,2,4]
203+-- @
204+--
205+-- Similarly, the 'Monad' instance interleaves the iterations of the
206+-- inner and the outer loop, nesting loops in a breadth first manner.
207+--
208+--
209+-- @
210+-- main = 'runStream' . 'wSerially' $ do
211+-- x <- return 1 \<\> return 2
212+-- y <- return 3 \<\> return 4
213+-- S.once $ print (x, y)
214+-- @
215+-- @
216+-- (1,3)
217+-- (2,3)
218+-- (1,4)
219+-- (2,4)
220+-- @
221+--
222+-- Note that a serial composition with breadth first traversal can only combine
223+-- a finite number of streams as it needs to retain state for each unfinished
224+-- stream.
225+--
226+-- @since 0.2.0
227+newtype WSerialT m a = WSerialT {getWSerialT :: Stream m a}
228+ deriving (Functor, MonadTrans)
229+
230+-- | An interleaving serial IO stream of elements of type @a@. See 'WSerialT'
231+-- documentation for more details.
232+--
233+-- @since 0.2.0
234+type WSerial a = WSerialT IO a
235+
236+-- |
237+-- @since 0.1.0
238+{-# DEPRECATED InterleavedT "Please use 'WSerialT' instead." #-}
239+type InterleavedT = WSerialT
240+
241+-- | Fix the type of a polymorphic stream as 'WSerialT'.
242+--
243+-- @since 0.2.0
244+wSerially :: IsStream t => WSerialT m a -> t m a
245+wSerially = adapt
246+
247+-- | Same as 'wSerially'.
248+--
249+-- @since 0.1.0
250+{-# DEPRECATED interleaving "Please use wSerially instead." #-}
251+interleaving :: IsStream t => WSerialT m a -> t m a
252+interleaving = wSerially
253+
254+instance IsStream WSerialT where
255+ toStream = getWSerialT
256+ fromStream = WSerialT
257+
258+ {-# INLINE consM #-}
259+ {-# SPECIALIZE consM :: IO a -> WSerialT IO a -> WSerialT IO a #-}
260+ consM :: Monad m => m a -> WSerialT m a -> WSerialT m a
261+ consM m r = fromStream $ consMSerial m (toStream r)
262+
263+ {-# INLINE (|:) #-}
264+ {-# SPECIALIZE (|:) :: IO a -> WSerialT IO a -> WSerialT IO a #-}
265+ (|:) :: Monad m => m a -> WSerialT m a -> WSerialT m a
266+ m |: r = fromStream $ consMSerial m (toStream r)
267+
268+------------------------------------------------------------------------------
269+-- Semigroup
270+------------------------------------------------------------------------------
271+
272+{-# INLINE interleave #-}
273+interleave :: Stream m a -> Stream m a -> Stream m a
274+interleave m1 m2 = Stream $ \_ stp sng yld -> do
275+ let stop = (runStream m2) Nothing stp sng yld
276+ single a = yld a m2
277+ yield a r = yld a (interleave m2 r)
278+ (runStream m1) Nothing stop single yield
279+
280+-- | Polymorphic version of the 'Semigroup' operation '<>' of 'WSerialT'.
281+-- Interleaves two streams, yielding one element from each stream alternately.
282+--
283+-- @since 0.2.0
284+{-# INLINE wSerial #-}
285+wSerial :: IsStream t => t m a -> t m a -> t m a
286+wSerial m1 m2 = fromStream $ interleave (toStream m1) (toStream m2)
287+
288+instance Semigroup (WSerialT m a) where
289+ (<>) = wSerial
290+
291+infixr 5 <=>
292+
293+-- | Same as 'wSerial'.
294+--
295+-- @since 0.1.0
296+{-# DEPRECATED (<=>) "Please use 'wSerial' instead." #-}
297+{-# INLINE (<=>) #-}
298+(<=>) :: IsStream t => t m a -> t m a -> t m a
299+(<=>) = wSerial
300+
301+------------------------------------------------------------------------------
302+-- Monoid
303+------------------------------------------------------------------------------
304+
305+instance Monoid (WSerialT m a) where
306+ mempty = nil
307+ mappend = (<>)
308+
309+------------------------------------------------------------------------------
310+-- Monad
311+------------------------------------------------------------------------------
312+
313+instance Monad m => Monad (WSerialT m) where
314+ return = pure
315+ (WSerialT (Stream m)) >>= f = WSerialT $ Stream $ \_ stp sng yld ->
316+ let runIt x = (runStream x) Nothing stp sng yld
317+ single a = runIt $ toStream (f a)
318+ yield a r = runIt $ toStream $ f a <> (fromStream r >>= f)
319+ in m Nothing stp single yield
320+
321+------------------------------------------------------------------------------
322+-- Other instances
323+------------------------------------------------------------------------------
324+
325+MONAD_APPLICATIVE_INSTANCE(WSerialT,)
326+MONAD_COMMON_INSTANCES(WSerialT,)
327+
328+-------------------------------------------------------------------------------
329+-- Running Streams
330+-------------------------------------------------------------------------------
331+
332+-- | Same as @runStream@.
333+--
334+-- @since 0.1.0
335+{-# DEPRECATED runStreamT "Please use runStream instead." #-}
336+runStreamT :: Monad m => SerialT m a -> m ()
337+runStreamT = run
338+
339+-- | Same as @runStream . wSerially@.
340+--
341+-- @since 0.1.0
342+{-# DEPRECATED runInterleavedT "Please use 'runStream . interleaving' instead." #-}
343+runInterleavedT :: Monad m => InterleavedT m a -> m ()
344+runInterleavedT = run
src/Streamly/Streams/Zip.hsnew file+255
@@ -0,0 +1,255 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE ConstraintKinds #-}
3+{-# LANGUAGE FlexibleContexts #-}
4+{-# LANGUAGE FlexibleInstances #-}
5+{-# LANGUAGE GeneralizedNewtypeDeriving#-}
6+{-# LANGUAGE InstanceSigs #-}
7+{-# LANGUAGE MultiParamTypeClasses #-}
8+{-# LANGUAGE StandaloneDeriving #-}
9+{-# LANGUAGE UndecidableInstances #-} -- XXX
10+
11+-- |
12+-- Module : Streamly.Streams.Zip
13+-- Copyright : (c) 2017 Harendra Kumar
14+--
15+-- License : BSD3
16+-- Maintainer : harendra.kumar@gmail.com
17+-- Stability : experimental
18+-- Portability : GHC
19+--
20+--
21+module Streamly.Streams.Zip
22+ (
23+ zipWith
24+ , zipWithM
25+ , zipAsyncWith
26+ , zipAsyncWithM
27+
28+ , ZipSerialM
29+ , ZipSerial
30+ , ZipStream -- deprecated
31+ , zipSerially
32+ , zipping -- deprecated
33+ , runZipStream -- deprecated
34+
35+ , ZipAsyncM
36+ , ZipAsync
37+ , zipAsyncly
38+ , zippingAsync -- deprecated
39+ , runZipAsync -- deprecated
40+ )
41+where
42+
43+import Data.Semigroup (Semigroup(..))
44+import Prelude hiding (repeat, zipWith)
45+
46+import Streamly.Streams.Async (mkAsync)
47+import Streamly.Streams.CPS
48+import Streamly.SVar (MonadAsync)
49+
50+#include "Instances.hs"
51+
52+------------------------------------------------------------------------------
53+-- Serial Zipping
54+------------------------------------------------------------------------------
55+
56+{-# INLINE zipWithS #-}
57+zipWithS :: (a -> b -> c) -> Stream m a -> Stream m b -> Stream m c
58+zipWithS f m1 m2 = go m1 m2
59+ where
60+ go mx my = Stream $ \_ stp sng yld -> do
61+ let merge a ra =
62+ let single2 b = sng (f a b)
63+ yield2 b rb = yld (f a b) (go ra rb)
64+ in (runStream my) Nothing stp single2 yield2
65+ let single1 a = merge a nil
66+ yield1 a ra = merge a ra
67+ (runStream mx) Nothing stp single1 yield1
68+
69+-- | Zip two streams serially using a pure zipping function.
70+--
71+-- @since 0.1.0
72+{-# INLINABLE zipWith #-}
73+zipWith :: IsStream t => (a -> b -> c) -> t m a -> t m b -> t m c
74+zipWith f m1 m2 = fromStream $ zipWithS f (toStream m1) (toStream m2)
75+
76+-- | Zip two streams serially using a monadic zipping function.
77+--
78+-- @since 0.1.0
79+zipWithM :: IsStream t => (a -> b -> t m c) -> t m a -> t m b -> t m c
80+zipWithM f m1 m2 = fromStream $ go (toStream m1) (toStream m2)
81+ where
82+ go mx my = Stream $ \_ stp sng yld -> do
83+ let merge a ra =
84+ let runIt x = runStream x Nothing stp sng yld
85+ single2 b = runIt $ toStream (f a b)
86+ yield2 b rb = runIt $ toStream (f a b) <> go ra rb
87+ in (runStream my) Nothing stp single2 yield2
88+ let single1 a = merge a nil
89+ yield1 a ra = merge a ra
90+ (runStream mx) Nothing stp single1 yield1
91+
92+------------------------------------------------------------------------------
93+-- Serially Zipping Streams
94+------------------------------------------------------------------------------
95+
96+-- | The applicative instance of 'ZipSerialM' zips a number of streams serially
97+-- i.e. it produces one element from each stream serially and then zips all
98+-- those elements.
99+--
100+-- @
101+-- main = (toList . 'zipSerially' $ (,,) \<$\> s1 \<*\> s2 \<*\> s3) >>= print
102+-- where s1 = fromFoldable [1, 2]
103+-- s2 = fromFoldable [3, 4]
104+-- s3 = fromFoldable [5, 6]
105+-- @
106+-- @
107+-- [(1,3,5),(2,4,6)]
108+-- @
109+--
110+-- The 'Semigroup' instance of this type works the same way as that of
111+-- 'SerialT'.
112+--
113+-- @since 0.2.0
114+newtype ZipSerialM m a = ZipSerialM {getZipSerialM :: Stream m a}
115+ deriving (Functor, Semigroup, Monoid)
116+
117+-- |
118+-- @since 0.1.0
119+{-# DEPRECATED ZipStream "Please use 'ZipSerialM' instead." #-}
120+type ZipStream = ZipSerialM
121+
122+-- | An IO stream whose applicative instance zips streams serially.
123+--
124+-- @since 0.2.0
125+type ZipSerial a = ZipSerialM IO a
126+
127+-- | Fix the type of a polymorphic stream as 'ZipSerialM'.
128+--
129+-- @since 0.2.0
130+zipSerially :: IsStream t => ZipSerialM m a -> t m a
131+zipSerially = adapt
132+
133+-- | Same as 'zipSerially'.
134+--
135+-- @since 0.1.0
136+{-# DEPRECATED zipping "Please use zipSerially instead." #-}
137+zipping :: IsStream t => ZipSerialM m a -> t m a
138+zipping = zipSerially
139+
140+instance IsStream ZipSerialM where
141+ toStream = getZipSerialM
142+ fromStream = ZipSerialM
143+
144+ {-# INLINE consM #-}
145+ {-# SPECIALIZE consM :: IO a -> ZipSerialM IO a -> ZipSerialM IO a #-}
146+ consM :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a
147+ consM m r = fromStream $ consMSerial m (toStream r)
148+
149+ {-# INLINE (|:) #-}
150+ {-# SPECIALIZE (|:) :: IO a -> ZipSerialM IO a -> ZipSerialM IO a #-}
151+ (|:) :: Monad m => m a -> ZipSerialM m a -> ZipSerialM m a
152+ m |: r = fromStream $ consMSerial m (toStream r)
153+
154+instance Monad m => Applicative (ZipSerialM m) where
155+ pure = ZipSerialM . repeat
156+ m1 <*> m2 = fromStream $ zipWith id (toStream m1) (toStream m2)
157+
158+------------------------------------------------------------------------------
159+-- Parallel Zipping
160+------------------------------------------------------------------------------
161+
162+-- | Zip two streams concurrently (i.e. both the elements being zipped are
163+-- generated concurrently) using a pure zipping function.
164+--
165+-- @since 0.1.0
166+zipAsyncWith :: (IsStream t, MonadAsync m)
167+ => (a -> b -> c) -> t m a -> t m b -> t m c
168+zipAsyncWith f m1 m2 = fromStream $ Stream $ \_ stp sng yld -> do
169+ ma <- mkAsync m1
170+ mb <- mkAsync m2
171+ (runStream (toStream (zipWith f ma mb))) Nothing stp sng yld
172+
173+-- | Zip two streams asyncly (i.e. both the elements being zipped are generated
174+-- concurrently) using a monadic zipping function.
175+--
176+-- @since 0.1.0
177+zipAsyncWithM :: (IsStream t, MonadAsync m)
178+ => (a -> b -> t m c) -> t m a -> t m b -> t m c
179+zipAsyncWithM f m1 m2 = fromStream $ Stream $ \_ stp sng yld -> do
180+ ma <- mkAsync m1
181+ mb <- mkAsync m2
182+ (runStream (toStream (zipWithM f ma mb))) Nothing stp sng yld
183+
184+------------------------------------------------------------------------------
185+-- Parallely Zipping Streams
186+------------------------------------------------------------------------------
187+--
188+-- | Like 'ZipSerialM' but zips in parallel, it generates all the elements to
189+-- be zipped concurrently.
190+--
191+-- @
192+-- main = (toList . 'zipAsyncly' $ (,,) \<$\> s1 \<*\> s2 \<*\> s3) >>= print
193+-- where s1 = fromFoldable [1, 2]
194+-- s2 = fromFoldable [3, 4]
195+-- s3 = fromFoldable [5, 6]
196+-- @
197+-- @
198+-- [(1,3,5),(2,4,6)]
199+-- @
200+--
201+-- The 'Semigroup' instance of this type works the same way as that of
202+-- 'SerialT'.
203+--
204+-- @since 0.2.0
205+newtype ZipAsyncM m a = ZipAsyncM {getZipAsyncM :: Stream m a}
206+ deriving (Functor, Semigroup, Monoid)
207+
208+-- | An IO stream whose applicative instance zips streams wAsyncly.
209+--
210+-- @since 0.2.0
211+type ZipAsync a = ZipAsyncM IO a
212+
213+-- | Fix the type of a polymorphic stream as 'ZipAsyncM'.
214+--
215+-- @since 0.2.0
216+zipAsyncly :: IsStream t => ZipAsyncM m a -> t m a
217+zipAsyncly = adapt
218+
219+-- | Same as 'zipAsyncly'.
220+--
221+-- @since 0.1.0
222+{-# DEPRECATED zippingAsync "Please use zipAsyncly instead." #-}
223+zippingAsync :: IsStream t => ZipAsyncM m a -> t m a
224+zippingAsync = zipAsyncly
225+instance IsStream ZipAsyncM where
226+ toStream = getZipAsyncM
227+ fromStream = ZipAsyncM
228+
229+ {-# INLINE consM #-}
230+ {-# SPECIALIZE consM :: IO a -> ZipAsyncM IO a -> ZipAsyncM IO a #-}
231+ consM :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a
232+ consM m r = fromStream $ consMSerial m (toStream r)
233+
234+ {-# INLINE (|:) #-}
235+ {-# SPECIALIZE (|:) :: IO a -> ZipAsyncM IO a -> ZipAsyncM IO a #-}
236+ (|:) :: Monad m => m a -> ZipAsyncM m a -> ZipAsyncM m a
237+ m |: r = fromStream $ consMSerial m (toStream r)
238+
239+instance MonadAsync m => Applicative (ZipAsyncM m) where
240+ pure = ZipAsyncM . repeat
241+ m1 <*> m2 = zipAsyncWith id m1 m2
242+
243+-- | Same as @runStream . zipping@.
244+--
245+-- @since 0.1.0
246+{-# DEPRECATED runZipStream "Please use 'runStream . zipSerially instead." #-}
247+runZipStream :: Monad m => ZipSerialM m a -> m ()
248+runZipStream = run
249+
250+-- | Same as @runStream . zippingAsync@.
251+--
252+-- @since 0.1.0
253+{-# DEPRECATED runZipAsync "Please use 'runStream . zipAsyncly instead." #-}
254+runZipAsync :: Monad m => ZipAsyncM m a -> m ()
255+runZipAsync = run
streamly.cabal+9−4
@@ -107,14 +107,19 @@ flag examples-sdl
107107library
108108 hs-source-dirs: src
109109 other-modules: Streamly.SVar
110− , Streamly.CPS.Stream
111− , Streamly.CPS.Concurrent
112− , Streamly.Streams
110+ , Streamly.Streams.CPS
111+ , Streamly.Streams.Serial
112+ , Streamly.Streams.SVar
113+ , Streamly.Streams.Async
114+ , Streamly.Streams.Parallel
115+ , Streamly.Streams.Ahead
116+ , Streamly.Streams.Zip
117+ , Streamly.Streams.Prelude
113118
114119 exposed-modules: Streamly.Prelude
115120 , Streamly.Time
116− , Streamly.Tutorial
117121 , Streamly
122+ , Streamly.Tutorial
118123
119124 default-language: Haskell2010
120125 ghc-options: -Wall