73c112d · streamly · git.iko.soy

git.iko.soy/streamly

Use spliceCopy instead of splice for immutable arrays

commit73c112d0ee38b78e6206bb47b56cf472d41f0a4c authorHarendra Kumar <harendra@composewell.com> dateTue 26 Dec 2023, 12:33 (3 years ago) parent23750f3

2 files changed+32−6

compare with master

unifiedsplit

core/src/Streamly/Internal/Data/Array/Type.hs+20−2 core/src/Streamly/Internal/Data/MutArray/Type.hs+12−4

core/src/Streamly/Internal/Data/Array/Type.hs+20−2

@@ -44,6 +44,7 @@ module Streamly.Internal.Data.Array.Type

4444 , fromByteStr#

4545 , fromByteStr

4646 , fromPtrN

47+ , fromArrayStreamK

4748

4849 -- ** Split

4950 , breakOn

@@ -117,6 +118,7 @@ import GHC.Ptr (Ptr(..))

117118import Streamly.Internal.Data.MutArray.Type (MutArray(..), MutByteArray)

118119import Streamly.Internal.Data.Fold.Type (Fold(..))

119120import Streamly.Internal.Data.Stream.Type (Stream)

121+import Streamly.Internal.Data.StreamK.Type (StreamK)

120122import Streamly.Internal.Data.Unbox (Unbox(..))

121123import Streamly.Internal.Data.Unfold.Type (Unfold(..))

122124import Text.Read (readPrec)

@@ -262,9 +264,9 @@ isPinned = MA.isPinned . unsafeThaw

262264-- would make a copy on every splice operation, instead use the

263265-- 'fromArrayStreamK' operation to combine n immutable arrays.

264266{-# INLINE splice #-}

265−splice :: (MonadIO m, Unbox a) => Array a -> Array a -> m (Array a)

267+splice :: MonadIO m => Array a -> Array a -> m (Array a)

266268splice arr1 arr2 =

267− unsafeFreeze <$> MA.splice (unsafeThaw arr1) (unsafeThaw arr2)

269+ unsafeFreeze <$> MA.spliceCopy (unsafeThaw arr1) (unsafeThaw arr2)

268270

269271-- | Create an 'Array' from the first N elements of a list. The array is

270272-- allocated to size N, if the list terminates before N elements then the

@@ -663,6 +665,16 @@ fromByteStr# addr = unsafePerformIO $ fmap unsafeFreeze (MA.fromByteStr# addr)

663665fromByteStr :: Ptr Word8 -> Array Word8

664666fromByteStr (Ptr addr#) = fromByteStr# addr#

665667

668+-- XXX implement fromArrayStream/fromArrayList instead?

669+

670+-- | Convert an array stream to an array. Note that this requires peak memory

671+-- that is double the size of the array stream.

672+--

673+{-# INLINE fromArrayStreamK #-}

674+fromArrayStreamK :: (Unbox a, MonadIO m) => StreamK m (Array a) -> m (Array a)

675+fromArrayStreamK stream =

676+ fmap unsafeFreeze $ MA.fromArrayStreamK $ fmap unsafeThaw stream

677+

666678-------------------------------------------------------------------------------

667679-- Instances

668680-------------------------------------------------------------------------------

@@ -815,6 +827,12 @@ instance Foldable Array where

815827-- Semigroup and Monoid

816828-------------------------------------------------------------------------------

817829

830+-- XXX Deprecate and remove the Semigroup and Monoid instances because of

831+-- potential misuse chances.

832+

833+-- | This should not be used for combining many or N arrays as it would copy

834+-- the two arrays everytime to a new array. For coalescing multiple arrays use

835+-- 'fromArrayStreamK' instead.

818836instance Unbox a => Semigroup (Array a) where

819837 arr1 <> arr2 = unsafePerformIO $ splice arr1 arr2

820838

core/src/Streamly/Internal/Data/MutArray/Type.hs+12−4

@@ -2087,21 +2087,29 @@ arrayStreamKLength as = K.foldl' (+) 0 (K.map length as)

20872087-- | Convert an array stream to an array. Note that this requires peak memory

20882088-- that is double the size of the array stream.

20892089--

2090−{-# INLINE fromArrayStreamK #-}

2091−fromArrayStreamK :: (Unbox a, MonadIO m) =>

2090+{-# INLINE fromArrayStreamKAs #-}

2091+fromArrayStreamKAs :: (Unbox a, MonadIO m) =>

20922092 PinnedState -> StreamK m (MutArray a) -> m (MutArray a)

2093−fromArrayStreamK ps as = do

2093+fromArrayStreamKAs ps as = do

20942094 len <- arrayStreamKLength as

20952095 arr <- newAs ps len

20962096 -- XXX is StreamK fold faster or StreamD fold?

20972097 K.foldlM' spliceUnsafe (pure arr) as

20982098 -- fromStreamDN len $ D.unfoldMany reader $ D.fromStreamK as

20992099

2100+-- | Convert an array stream to an array. Note that this requires peak memory

2101+-- that is double the size of the array stream.

2102+--

2103+{-# INLINE fromArrayStreamK #-}

2104+fromArrayStreamK :: (Unbox a, MonadIO m) =>

2105+ StreamK m (MutArray a) -> m (MutArray a)

2106+fromArrayStreamK = fromArrayStreamKAs Unpinned

2107+

21002108{-# INLINE fromStreamDAs #-}

21012109fromStreamDAs ::

21022110 (MonadIO m, Unbox a) => PinnedState -> D.Stream m a -> m (MutArray a)

21032111fromStreamDAs ps m =

2104− arrayStreamKFromStreamDAs Unpinned m >>= fromArrayStreamK ps

2112+ arrayStreamKFromStreamDAs Unpinned m >>= fromArrayStreamKAs ps

21052113

21062114-- CAUTION: a very large number (millions) of arrays can degrade performance

21072115-- due to GC overhead because we need to buffer the arrays before we flatten