git.iko.soy/streamly
on master help sign in
Use array copy (memcpy) instead of stream copy

For better performance.

commit7a8b3627e7f94c8e1921a240f8a4db31267503e3 authorHarendra Kumar <harendra@composewell.com> dateTue 26 Dec 2023, 09:14 (3 years ago) parent090d1d8
3 files changed+28−25
compare with master
unifiedsplit
core/src/Streamly/Internal/Data/MutArray/Type.hs+8−5 core/src/Streamly/Internal/Data/StreamK.hs−20 core/src/Streamly/Internal/Data/StreamK/Type.hs+20
core/src/Streamly/Internal/Data/MutArray/Type.hs+8−5
@@ -1383,7 +1383,6 @@ pinnedChunksOf :: forall m a. (MonadIO m, Unbox a)
13831383-- pinnedChunksOf n = D.foldMany (pinnedWriteN n)
13841384pinnedChunksOf = chunksOfAs Pinned
13851385
1386−-- XXX This should take a PinnedState
13871386-- XXX buffer to a list instead?
13881387-- | Buffer the stream into arrays in memory.
13891388{-# INLINE arrayStreamKFromStreamDAs #-}
@@ -2088,15 +2087,19 @@ arrayStreamKLength as = K.foldl' (+) 0 (K.map length as)
20882087--
20892088{-# INLINE fromArrayStreamK #-}
20902089fromArrayStreamK :: (Unbox a, MonadIO m) =>
2091− StreamK m (MutArray a) -> m (MutArray a)
2092−fromArrayStreamK as = do
2090+ PinnedState -> StreamK m (MutArray a) -> m (MutArray a)
2091+fromArrayStreamK ps as = do
20932092 len <- arrayStreamKLength as
2094− fromStreamDN len $ D.unfoldMany reader $ D.fromStreamK as
2093+ arr <- newAs ps len
2094+ -- XXX is StreamK fold faster or StreamD fold?
2095+ K.foldlM' spliceUnsafe (pure arr) as
2096+ -- fromStreamDN len $ D.unfoldMany reader $ D.fromStreamK as
20952097
20962098{-# INLINE fromStreamDAs #-}
20972099fromStreamDAs ::
20982100 (MonadIO m, Unbox a) => PinnedState -> D.Stream m a -> m (MutArray a)
2099−fromStreamDAs ps m = arrayStreamKFromStreamDAs ps m >>= fromArrayStreamK
2101+fromStreamDAs ps m =
2102+ arrayStreamKFromStreamDAs Unpinned m >>= fromArrayStreamK ps
21002103
21012104-- CAUTION: a very large number (millions) of arrays can degrade performance
21022105-- due to GC overhead because we need to buffer the arrays before we flatten
core/src/Streamly/Internal/Data/StreamK.hs−20
@@ -37,8 +37,6 @@ module Streamly.Internal.Data.StreamK
3737 -- * Elimination
3838 -- ** General Folds
3939 , foldr1
40− , foldlM'
41− , foldlMx'
4240 , fold
4341 , foldBreak
4442 , foldEither
@@ -306,19 +304,6 @@ foldr1 step m = do
306304 yieldk a r = fmap (step p) (go a r)
307305 in foldStream defState yieldk single stp m1
308306
309−-- XXX replace the recursive "go" with explicit continuations.
310−-- | Like 'foldx', but with a monadic step function.
311−{-# INLINABLE foldlMx' #-}
312−foldlMx' :: Monad m
313− => (x -> a -> m x) -> m x -> (x -> m b) -> StreamK m a -> m b
314−foldlMx' step begin done = go begin
315− where
316− go !acc m1 =
317− let stop = acc >>= done
318− single a = acc >>= \b -> step b a >>= done
319− yieldk a r = acc >>= \b -> step b a >>= \x -> go (return x) r
320− in foldStream defState yieldk single stop m1
321−
322307-- | Fold a stream using the supplied left 'Fold' and reducing the resulting
323308-- expression strictly at each step. The behavior is similar to 'foldl''. A
324309-- 'Fold' can terminate early without consuming the full stream. See the
@@ -470,11 +455,6 @@ foldConcat
470455 Stream.Skip s -> go1 SPEC fs s
471456 Stream.Stop -> return $ Left fs
472457
473−-- | Like 'foldl'' but with a monadic step function.
474−{-# INLINE foldlM' #-}
475−foldlM' :: Monad m => (b -> a -> m b) -> m b -> StreamK m a -> m b
476−foldlM' step begin = foldlMx' step begin return
477−
478458------------------------------------------------------------------------------
479459-- Specialized folds
480460------------------------------------------------------------------------------
core/src/Streamly/Internal/Data/StreamK/Type.hs+20
@@ -84,6 +84,8 @@ module Streamly.Internal.Data.StreamK.Type
8484 -- ** Strict Left Folds
8585 , Streamly.Internal.Data.StreamK.Type.foldl'
8686 , foldlx'
87+ , foldlMx'
88+ , foldlM'
8789
8890 -- ** Lazy Right Folds
8991 , Streamly.Internal.Data.StreamK.Type.foldr
@@ -817,6 +819,24 @@ foldlx' step begin done m = get $ go m begin
817819foldl' :: Monad m => (b -> a -> b) -> b -> StreamK m a -> m b
818820foldl' step begin = foldlx' step begin id
819821
822+-- XXX replace the recursive "go" with explicit continuations.
823+-- | Like 'foldx', but with a monadic step function.
824+{-# INLINABLE foldlMx' #-}
825+foldlMx' :: Monad m
826+ => (x -> a -> m x) -> m x -> (x -> m b) -> StreamK m a -> m b
827+foldlMx' step begin done = go begin
828+ where
829+ go !acc m1 =
830+ let stop = acc >>= done
831+ single a = acc >>= \b -> step b a >>= done
832+ yieldk a r = acc >>= \b -> step b a >>= \x -> go (return x) r
833+ in foldStream defState yieldk single stop m1
834+
835+-- | Like 'foldl'' but with a monadic step function.
836+{-# INLINE foldlM' #-}
837+foldlM' :: Monad m => (b -> a -> m b) -> m b -> StreamK m a -> m b
838+foldlM' step begin = foldlMx' step begin return
839+
820840------------------------------------------------------------------------------
821841-- Specialized folds
822842------------------------------------------------------------------------------