@@ -40,22 +40,22 @@ module Streamly.Internal.Data.MutArray.Type
4040
4141 -- ** Casting
4242 , cast
43− , castUnsafe
43+ , castUnsafe -- XXX unsafeCast
4444 , asBytes
45− , asPtrUnsafe
46− , asUnpinnedPtrUnsafe
45+ , unsafePinnedAsPtr
46+ , unsafeAsPtr
4747
4848 -- ** Construction
49− , nil
49+ , empty
5050
5151 -- *** New
5252 -- | New arrays are always empty arrays with some reserve capacity to
5353 -- extend the length without reallocating.
54− , new
55− , newArrayWith
56− , pinnedNew
57− , pinnedNewBytes
58− , pinnedNewAligned
54+ , emptyOf
55+ , newArrayWith -- emptyAlignedWith
56+ , pinnedEmptyOf
57+ , pinnedNewAligned -- XXX not required
58+ -- , new -- uninitialized array of specified length
5959
6060 -- *** Cloning
6161 , clone
@@ -63,27 +63,25 @@ module Streamly.Internal.Data.MutArray.Type
6363
6464 -- *** Slicing
6565 -- | Get a subarray without copying
66− , getSliceUnsafe
66+ , getSliceUnsafe -- XXX unsafeGetSlice
6767 , getSlice
6868 , splitAt -- XXX should be able to express using getSlice
6969 , breakOn
7070
7171 -- *** Stream Folds
7272 , ArrayUnsafe (..)
73− , writeNWithUnsafe
74− , writeNWith
75− , writeNUnsafe
76− , pinnedWriteNUnsafe
77− , writeN
78− , pinnedWriteN
79− , pinnedWriteNAligned
80−
81− , writeWith
82− , write
83− , pinnedWrite
84−
85− , writeRevN
86− -- , writeRev
73+ , unsafeCreateOfWith
74+ , unsafeCreateOf
75+ , unsafePinnedCreateOf
76+ , pinnedCreateOf
77+ , createOfWith
78+ , createOf
79+ , revCreateOf
80+
81+ , pinnedCreate
82+ , createWith
83+ , create
84+ -- , revCreate
8785
8886 -- *** From containers
8987 , fromListN
@@ -99,11 +97,11 @@ module Streamly.Internal.Data.MutArray.Type
9997 , fromByteStr#
10098 , fromPtrN
10199 , fromChunksK
102− , fromChunksRealloced
100+ , fromChunksRealloced -- fromSmallChunks
103101
104102 -- ** Random writes
105103 , putIndex
106− , putIndexUnsafe
104+ , putIndexUnsafe -- XXX unsafePutIndex
107105 , putIndices
108106 -- , putFromThenTo
109107 -- , putFrom -- start writing at the given position
@@ -111,7 +109,7 @@ module Streamly.Internal.Data.MutArray.Type
111109 -- , putFromTo
112110 -- , putFromRev
113111 -- , putUptoRev
114− , modifyIndexUnsafe
112+ , modifyIndexUnsafe -- XXX unsafeModifyIndex
115113 , modifyIndex
116114 , modifyIndices
117115 , modify
@@ -122,9 +120,9 @@ module Streamly.Internal.Data.MutArray.Type
122120
123121 -- *** Indexing
124122 , getIndex
125− , getIndexUnsafe
123+ , getIndexUnsafe -- XXX unsafeGetIndex
126124 -- , getFromThenTo
127− , getIndexRev
125+ , getIndexRev -- getRevIndex?
128126 , indexReader
129127 , indexReaderWith
130128
@@ -164,9 +162,9 @@ module Streamly.Internal.Data.MutArray.Type
164162 , blockSize
165163 , arrayChunkBytes
166164 , allocBytesToElemCount
167− , realloc
168− , resize
169− , resizeExp
165+ , realloc -- this is "resize"
166+ , grow
167+ , growExp
170168 , rightSize
171169
172170 -- ** Folding
@@ -195,24 +193,34 @@ module Streamly.Internal.Data.MutArray.Type
195193 , snoc
196194 , snocLinear
197195 , snocMay
198− , snocUnsafe
196+ , snocUnsafe -- XXX unsafeSnoc
199197
200198 -- *** Appending streams
201− , writeAppendNUnsafe
202− , writeAppendN
203− , writeAppendWith
204− , writeAppend
199+ , unsafeAppendN
200+ , appendN
201+ , appendWith
202+ , append
205203
206204 -- *** Appending arrays
207205 , spliceCopy
208206 , spliceWith
209207 , splice
210208 , spliceExp
211− , spliceUnsafe
209+ , spliceUnsafe -- XXX unsafeSplice
212210 -- , putSlice
213211 -- , appendSlice
214212 -- , appendSliceFrom
215213
214+ -- ** Serialization using Unbox
215+ , pokeAppend
216+ , pokeAppendMay
217+ , pokeSkipUnsafe -- unsafePokeSkip
218+
219+ -- ** Deserialization using Unbox
220+ , peekUncons
221+ , peekUnconsUnsafe -- unsafePeekUncons
222+ , peekSkipUnsafe -- unsafePeekSkip
223+
216224 -- Arrays of arrays
217225 -- We can add dimensionality parameter to the array type to get
218226 -- multidimensional arrays. Multidimensional arrays would just be a
@@ -234,6 +242,8 @@ module Streamly.Internal.Data.MutArray.Type
234242
235243 -- *** Concat
236244 -- | Append the arrays in a stream to form a stream of elements.
245+ , concatWith
246+ , concatRevWith
237247 , concat
238248 , concatRev
239249
@@ -241,9 +251,12 @@ module Streamly.Internal.Data.MutArray.Type
241251 -- | Append the arrays in a stream to form a stream of larger arrays.
242252 , SpliceState (..)
243253 , pCompactLE
244− , rCompactLE
254+ , pPinnedCompactLE
255+ , compactLeAs
245256 , fCompactGE
257+ , fPinnedCompactGE
246258 , lCompactGE
259+ , lPinnedCompactGE
247260 , compactGE
248261 , compactEQ
249262
@@ -254,6 +267,7 @@ module Streamly.Internal.Data.MutArray.Type
254267 , c_memchr
255268
256269 -- * Deprecated
270+ , asPtrUnsafe
257271 , writeChunks
258272 , flattenArrays
259273 , flattenArraysRev
@@ -263,6 +277,27 @@ module Streamly.Internal.Data.MutArray.Type
263277 , cmp
264278 , getIndices
265279 , getIndicesWith
280+ , resize
281+ , resizeExp
282+ , nil
283+ , new
284+ , pinnedNew
285+ , pinnedNewBytes
286+ , writeAppendNUnsafe
287+ , writeAppendN
288+ , writeAppendWith
289+ , writeAppend
290+ , writeNWithUnsafe
291+ , writeNWith
292+ , writeNUnsafe
293+ , pinnedWriteNUnsafe
294+ , writeN
295+ , pinnedWriteN
296+ , pinnedWriteNAligned -- XXX not required
297+ , writeWith
298+ , write
299+ , pinnedWrite
300+ , writeRevN
266301 )
267302where
268303
@@ -444,6 +479,8 @@ isPinned MutArray{..} = Unboxed.isPinned arrContents
444479-- we need more than that. See stg_pinnedNewAlignedByteArrayzh and
445480-- allocatePinned in GHC source.
446481
482+-- XXX Rename to emptyAlignedWith, alignSize should be first arg.
483+
447484-- | @newArrayWith allocator alignment count@ allocates a new array of zero
448485-- length and with a capacity to hold @count@ elements, using @allocator
449486-- size alignment@ as the memory allocator function.
@@ -467,12 +504,25 @@ newArrayWith alloc alignSize count = do
467504 , arrBound = size
468505 }
469506
507+-- For arrays "nil" sounds a bit odd. empty is better. The only problem with
508+-- empty is that it is also used by the Alternative type class. But assuming we
509+-- will mostly import the Array module qualified this should be fine.
510+
511+-- | Create an empty array.
512+empty ::
513+#ifdef DEVBUILD
514+ Unbox a =>
515+#endif
516+ MutArray a
517+empty = MutArray Unboxed.empty 0 0 0
518+
519+{-# DEPRECATED nil "Please use empty instead." #-}
470520nil ::
471521#ifdef DEVBUILD
472522 Unbox a =>
473523#endif
474524 MutArray a
475−nil = MutArray Unboxed.nil 0 0 0
525+nil = empty
476526
477527{-# INLINE newBytesAs #-}
478528newBytesAs :: MonadIO m =>
@@ -489,12 +539,13 @@ newBytesAs ps bytes = do
489539 , arrBound = bytes
490540 }
491541
492−-- | Allocates a pinned empty array that can hold 'count' items. The memory of
493−-- the array is uninitialized and the allocation is aligned as per the
494−-- 'Unboxed' instance of the type.
542+-- | Allocates a pinned empty array that with a reserved capacity of bytes.
543+-- The memory of the array is uninitialized and the allocation is aligned as
544+-- per the 'Unboxed' instance of the type.
495545--
496546-- /Pre-release/
497547{-# INLINE pinnedNewBytes #-}
548+{-# DEPRECATED pinnedNewBytes "Please use pinnedEmptyOf with appropriate calculation" #-}
498549pinnedNewBytes :: MonadIO m =>
499550#ifdef DEVBUILD
500551 Unbox a =>
@@ -519,21 +570,31 @@ newAs ps =
519570 (error "new: alignment is not used in unpinned arrays.")
520571
521572-- XXX can unaligned allocation be more efficient when alignment is not needed?
522−--
523−-- | Allocates an empty pinned array that can hold 'count' items. The memory of
524−-- the array is uninitialized and the allocation is aligned as per the 'Unboxed'
525−-- instance of the type.
526−--
573+
574+-- | Allocates a pinned array of zero length but growable to the specified
575+-- capacity without reallocation.
576+{-# INLINE pinnedEmptyOf #-}
577+pinnedEmptyOf :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)
578+pinnedEmptyOf = newAs Pinned
579+
580+-- XXX Deprecate in major
581+-- {-# DEPRECATED pinnedNew "Please use pinnedEmptyOf instead." #-}
527582{-# INLINE pinnedNew #-}
528583pinnedNew :: forall m a. (MonadIO m, Unbox a) => Int -> m (MutArray a)
529−pinnedNew = newAs Pinned
584+pinnedNew = pinnedEmptyOf
530585
531−-- | Allocates an empty unpinned array that can hold 'count' items. The memory
532−-- of the array is uninitialized.
586+-- | Allocates an unpinned array of zero length but growable to the specified
587+-- capacity without reallocation.
533588--
589+{-# INLINE emptyOf #-}
590+emptyOf :: (MonadIO m, Unbox a) => Int -> m (MutArray a)
591+emptyOf = newAs Unpinned
592+
593+-- XXX Deprecate in major
594+-- {-# DEPRECATED new "Please use emptyOf instead." #-}
534595{-# INLINE new #-}
535596new :: (MonadIO m, Unbox a) => Int -> m (MutArray a)
536−new = newAs Unpinned
597+new = emptyOf
537598
538599-------------------------------------------------------------------------------
539600-- Random writes
@@ -781,15 +842,15 @@ roundDownTo elemSize size = size - (size `mod` elemSize)
781842-- Since this is not inlined Unboxed consrraint leads to dictionary passing
782843-- which complicates some inspection tests.
783844--
784−{-# NOINLINE reallocExplicit #-}
785−reallocExplicit :: Int -> Int -> MutArray a -> IO (MutArray a)
786−reallocExplicit elemSize newCapacityInBytes MutArray{..} = do
845+{-# NOINLINE reallocExplicitAs #-}
846+reallocExplicitAs :: PinnedState -> Int -> Int -> MutArray a -> IO (MutArray a)
847+reallocExplicitAs ps elemSize newCapacityInBytes MutArray{..} = do
787848 assertM(arrEnd <= arrBound)
788849
789850 -- Allocate new array
790851 let newCapMaxInBytes = roundUpLargeArray newCapacityInBytes
791852 contents <-
792− if Unboxed.isPinned arrContents
853+ if ps == Pinned
793854 then Unboxed.pinnedNew newCapMaxInBytes
794855 else Unboxed.new newCapMaxInBytes
795856 let !(MutByteArray mbarrFrom#) = arrContents
@@ -829,7 +890,12 @@ reallocExplicit elemSize newCapacityInBytes MutArray{..} = do
829890-- If the original array is pinned, the newly allocated array is also pinned.
830891{-# INLINABLE realloc #-}
831892realloc :: forall m a. (MonadIO m, Unbox a) => Int -> MutArray a -> m (MutArray a)
832−realloc bytes arr = liftIO $ reallocExplicit (SIZE_OF(a)) bytes arr
893+realloc bytes arr =
894+ let ps =
895+ if isPinned arr
896+ then Pinned
897+ else Unpinned
898+ in liftIO $ reallocExplicitAs ps (SIZE_OF(a)) bytes arr
833899
834900-- | @reallocWith label capSizer minIncrBytes array@. The label is used
835901-- in error messages and the capSizer is used to determine the capacity of the
@@ -859,9 +925,7 @@ reallocWith label capSizer minIncrBytes arr = do
859925 , ". Please check the sizing function passed."
860926 ]
861927
862−-- XXX should be called expand/extend because it never shortens the array.
863−
864−-- | @resize newCapacity array@ changes the total capacity of the array so that
928+-- | @grow newCapacity array@ changes the total capacity of the array so that
865929-- it is enough to hold the specified number of elements. Nothing is done if
866930-- the specified capacity is less than the length of the array.
867931--
@@ -869,24 +933,30 @@ reallocWith label capSizer minIncrBytes arr = do
869933-- the block size (4K).
870934--
871935-- /Pre-release/
872−{-# INLINE resize #-}
873−resize :: forall m a. (MonadIO m, Unbox a) =>
936+{-# INLINE grow #-}
937+grow :: forall m a. (MonadIO m, Unbox a) =>
874938 Int -> MutArray a -> m (MutArray a)
875−resize nElems arr@MutArray{..} = do
939+grow nElems arr@MutArray{..} = do
876940 let req = SIZE_OF(a) * nElems
877941 cap = arrBound - arrStart
878942 if req < cap
879943 then return arr
880944 else realloc req arr
881945
882−-- | Like 'resize' but if the requested byte capacity is more than
946+{-# DEPRECATED resize "Please use grow instead." #-}
947+{-# INLINE resize #-}
948+resize :: forall m a. (MonadIO m, Unbox a) =>
949+ Int -> MutArray a -> m (MutArray a)
950+resize = grow
951+
952+-- | Like 'grow' but if the requested byte capacity is more than
883953-- 'largeObjectThreshold' then it is rounded up to the closest power of 2.
884954--
885955-- /Pre-release/
886−{-# INLINE resizeExp #-}
887−resizeExp :: forall m a. (MonadIO m, Unbox a) =>
956+{-# INLINE growExp #-}
957+growExp :: forall m a. (MonadIO m, Unbox a) =>
888958 Int -> MutArray a -> m (MutArray a)
889−resizeExp nElems arr@MutArray{..} = do
959+growExp nElems arr@MutArray{..} = do
890960 let req = roundUpLargeArray (SIZE_OF(a) * nElems)
891961 req1 =
892962 if req > largeObjectThreshold
@@ -897,6 +967,12 @@ resizeExp nElems arr@MutArray{..} = do
897967 then return arr
898968 else realloc req1 arr
899969
970+{-# DEPRECATED resizeExp "Please use growExp instead." #-}
971+{-# INLINE resizeExp #-}
972+resizeExp :: forall m a. (MonadIO m, Unbox a) =>
973+ Int -> MutArray a -> m (MutArray a)
974+resizeExp = growExp
975+
900976-- | Resize the allocated memory to drop any reserved free space at the end of
901977-- the array and reallocate it to reduce wastage.
902978--
@@ -1041,6 +1117,126 @@ snoc = snocWith f
10411117 then oldSize * 2
10421118 else roundUpToPower2 oldSize * 2
10431119
1120+-------------------------------------------------------------------------------
1121+-- Serialization/Deserialization
1122+-------------------------------------------------------------------------------
1123+
1124+{-# INLINE pokeNewEnd #-}
1125+pokeNewEnd :: (MonadIO m, Unbox a) =>
1126+ Int -> MutArray Word8 -> a -> m (MutArray Word8)
1127+pokeNewEnd newEnd arr@MutArray{..} x = liftIO $ do
1128+ assert (newEnd <= arrBound) (return ())
1129+ liftIO $ pokeAt arrEnd arrContents x
1130+ return $ arr {arrEnd = newEnd}
1131+
1132+-- | Really really unsafe, unboxes a Haskell type and appends the resulting
1133+-- bytes to the byte array, may cause silent data corruption or if you are
1134+-- lucky a segfault if the array does not have enough space to append the
1135+-- element.
1136+--
1137+-- /Internal/
1138+{-# INLINE pokeAppendUnsafe #-}
1139+pokeAppendUnsafe :: forall m a. (MonadIO m, Unbox a) =>
1140+ MutArray Word8 -> a -> m (MutArray Word8)
1141+pokeAppendUnsafe arr@MutArray{..} = pokeNewEnd (arrEnd + SIZE_OF(a)) arr
1142+
1143+-- | Skip the specified number of bytes in the array. The data in the skipped
1144+-- region remains uninitialzed.
1145+{-# INLINE pokeSkipUnsafe #-}
1146+pokeSkipUnsafe :: Int -> MutArray Word8 -> MutArray Word8
1147+pokeSkipUnsafe n arr@MutArray{..} = do
1148+ let newEnd = arrEnd + n
1149+ in assert (newEnd <= arrBound) (arr {arrEnd = newEnd})
1150+
1151+-- | Like 'pokeAppend' but does not grow the array when pre-allocated array
1152+-- capacity becomes full.
1153+--
1154+-- /Internal/
1155+{-# INLINE pokeAppendMay #-}
1156+pokeAppendMay :: forall m a. (MonadIO m, Unbox a) =>
1157+ MutArray Word8 -> a -> m (Maybe (MutArray Word8))
1158+pokeAppendMay arr@MutArray{..} x = liftIO $ do
1159+ let newEnd = arrEnd + SIZE_OF(a)
1160+ if newEnd <= arrBound
1161+ then Just <$> pokeNewEnd newEnd arr x
1162+ else return Nothing
1163+
1164+{-# NOINLINE pokeWithRealloc #-}
1165+pokeWithRealloc :: forall m a. (MonadIO m, Unbox a) =>
1166+ (Int -> Int)
1167+ -> MutArray Word8
1168+ -> a
1169+ -> m (MutArray Word8)
1170+pokeWithRealloc sizer arr x = do
1171+ arr1 <- liftIO $ reallocWith "pokeWithRealloc" sizer (SIZE_OF(a)) arr
1172+ pokeAppendUnsafe arr1 x
1173+
1174+{-# INLINE pokeAppendWith #-}
1175+pokeAppendWith :: forall m a. (MonadIO m, Unbox a) =>
1176+ (Int -> Int)
1177+ -> MutArray Word8
1178+ -> a
1179+ -> m (MutArray Word8)
1180+pokeAppendWith allocSize arr x = liftIO $ do
1181+ let newEnd = arrEnd arr + SIZE_OF(a)
1182+ if newEnd <= arrBound arr
1183+ then pokeNewEnd newEnd arr x
1184+ else pokeWithRealloc allocSize arr x
1185+
1186+-- | Unbox a Haskell type and append the resulting bytes to a mutable byte
1187+-- array. The array is grown exponentially when more space is needed.
1188+--
1189+-- Definition:
1190+--
1191+-- >>> pokeAppend arr x = MutArray.castUnsafe <$> MutArray.snoc (MutArray.castUnsafe arr) x
1192+--
1193+{-# INLINE pokeAppend #-}
1194+pokeAppend :: forall m a. (MonadIO m, Unbox a) =>
1195+ MutArray Word8 -> a -> m (MutArray Word8)
1196+pokeAppend = pokeAppendWith f
1197+
1198+ where
1199+
1200+ f oldSize =
1201+ if isPower2 oldSize
1202+ then oldSize * 2
1203+ else roundUpToPower2 oldSize * 2
1204+
1205+-- | Really really unsafe, create a Haskell value from an unboxed byte array,
1206+-- does not check if the array is big enough, may return garbage or if you are
1207+-- lucky may cause a segfault.
1208+--
1209+-- /Internal/
1210+{-# INLINE peekUnconsUnsafe #-}
1211+peekUnconsUnsafe :: forall m a. (MonadIO m, Unbox a) =>
1212+ MutArray Word8 -> m (a, MutArray Word8)
1213+peekUnconsUnsafe MutArray{..} = do
1214+ let start1 = arrStart + SIZE_OF(a)
1215+ assert (start1 <= arrEnd) (return ())
1216+ liftIO $ do
1217+ r <- peekAt arrStart arrContents
1218+ return (r, MutArray arrContents start1 arrEnd arrBound)
1219+
1220+-- | Discard the specified number of bytes in the array.
1221+{-# INLINE peekSkipUnsafe #-}
1222+peekSkipUnsafe :: Int -> MutArray Word8 -> MutArray Word8
1223+peekSkipUnsafe n MutArray{..} =
1224+ let start1 = arrStart + n
1225+ in assert (start1 <= arrEnd) (MutArray arrContents start1 arrEnd arrBound)
1226+
1227+-- | Create a Haskell value from its unboxed representation from the head of a
1228+-- byte array, return the value and the remaining array.
1229+{-# INLINE peekUncons #-}
1230+peekUncons :: forall m a. (MonadIO m, Unbox a) =>
1231+ MutArray Word8 -> m (Maybe a, MutArray Word8)
1232+peekUncons arr@MutArray{..} = do
1233+ let start1 = arrStart + SIZE_OF(a)
1234+ if start1 > arrEnd
1235+ then return (Nothing, arr)
1236+ else liftIO $ do
1237+ r <- peekAt arrStart arrContents
1238+ return (Just r, MutArray arrContents start1 arrEnd arrBound)
1239+
10441240-------------------------------------------------------------------------------
10451241-- Random reads
10461242-------------------------------------------------------------------------------
@@ -1106,6 +1302,7 @@ indexReaderWith liftio (D.Stream stepi sti) = Unfold step inject
11061302 D.Stop -> return D.Stop
11071303
11081304{-# DEPRECATED getIndicesWith "Please use indexReaderWith instead." #-}
1305+{-# INLINE getIndicesWith #-}
11091306getIndicesWith :: (Monad m, Unbox a) =>
11101307 (forall b. IO b -> m b) -> D.Stream m Int -> Unfold m (MutArray a) a
11111308getIndicesWith = indexReaderWith
@@ -1121,6 +1318,7 @@ indexReader = indexReaderWith liftIO
11211318
11221319-- XXX DO NOT REMOVE, change the signature to use Stream instead of unfold
11231320{-# DEPRECATED getIndices "Please use indexReader instead." #-}
1321+{-# INLINE getIndices #-}
11241322getIndices :: (MonadIO m, Unbox a) => Stream m Int -> Unfold m (MutArray a) a
11251323getIndices = indexReader
11261324
@@ -1417,7 +1615,7 @@ chunksOfAs ps n (D.Stream step state) =
14171615--
14181616-- Same as the following but may be more efficient:
14191617--
1420−-- >>> chunksOf n = Stream.foldMany (MutArray.writeN n)
1618+-- >>> chunksOf n = Stream.foldMany (MutArray.createOf n)
14211619--
14221620-- /Pre-release/
14231621{-# INLINE_NORMAL chunksOf #-}
@@ -1426,14 +1624,14 @@ chunksOf :: forall m a. (MonadIO m, Unbox a)
14261624-- XXX the idiomatic implementation leads to large regression in the D.reverse'
14271625-- benchmark. It seems it has difficulty producing optimized code when
14281626-- converting to StreamK. Investigate GHC optimizations.
1429−-- chunksOf n = D.foldMany (writeN n)
1627+-- chunksOf n = D.foldMany (createOf n)
14301628chunksOf = chunksOfAs Unpinned
14311629
14321630-- | Like 'chunksOf' but creates pinned arrays.
14331631{-# INLINE_NORMAL pinnedChunksOf #-}
14341632pinnedChunksOf :: forall m a. (MonadIO m, Unbox a)
14351633 => Int -> D.Stream m a -> D.Stream m (MutArray a)
1436−-- pinnedChunksOf n = D.foldMany (pinnedWriteN n)
1634+-- pinnedChunksOf n = D.foldMany (pinnedCreateOf n)
14371635pinnedChunksOf = chunksOfAs Pinned
14381636
14391637-- | When we are buffering a stream of unknown size into an array we do not
@@ -1466,16 +1664,10 @@ data FlattenState s contents a =
14661664 OuterLoop s
14671665 | InnerLoop s contents !Int !Int
14681666
1469−-- | Use the "reader" unfold instead.
1470−--
1471−-- @concat = unfoldMany reader@
1472−--
1473−-- We can try this if there are any fusion issues in the unfold.
1474−--
1475−{-# INLINE_NORMAL concat #-}
1476−concat :: forall m a. (MonadIO m, Unbox a)
1477− => D.Stream m (MutArray a) -> D.Stream m a
1478−concat (D.Stream step state) = D.Stream step' (OuterLoop state)
1667+{-# INLINE_NORMAL concatWith #-}
1668+concatWith :: forall m a. (Monad m, Unbox a)
1669+ => (forall b. IO b -> m b) -> D.Stream m (MutArray a) -> D.Stream m a
1670+concatWith liftio (D.Stream step state) = D.Stream step' (OuterLoop state)
14791671
14801672 where
14811673
@@ -1492,24 +1684,30 @@ concat (D.Stream step state) = D.Stream step' (OuterLoop state)
14921684 return $ D.Skip $ OuterLoop st
14931685
14941686 step' _ (InnerLoop st contents p end) = do
1495− x <- liftIO $ peekAt p contents
1687+ x <- liftio $ peekAt p contents
14961688 return $ D.Yield x (InnerLoop st contents (INDEX_NEXT(p,a)) end)
14971689
1498−{-# DEPRECATED flattenArrays "Please use \"unfoldMany reader\" instead." #-}
1499−flattenArrays :: forall m a. (MonadIO m, Unbox a)
1500− => D.Stream m (MutArray a) -> D.Stream m a
1501−flattenArrays = concat
1502−
1503−-- | Use the "readerRev" unfold instead.
1690+-- | Use the "reader" unfold instead.
15041691--
1505−-- @concat = unfoldMany readerRev@
1692+-- @concat = unfoldMany reader@
15061693--
15071694-- We can try this if there are any fusion issues in the unfold.
15081695--
1509−{-# INLINE_NORMAL concatRev #-}
1510−concatRev :: forall m a. (MonadIO m, Unbox a)
1696+{-# INLINE_NORMAL concat #-}
1697+concat :: forall m a. (MonadIO m, Unbox a)
15111698 => D.Stream m (MutArray a) -> D.Stream m a
1512−concatRev (D.Stream step state) = D.Stream step' (OuterLoop state)
1699+concat = concatWith liftIO
1700+
1701+{-# DEPRECATED flattenArrays "Please use \"unfoldMany reader\" instead." #-}
1702+{-# INLINE flattenArrays #-}
1703+flattenArrays :: forall m a. (MonadIO m, Unbox a)
1704+ => D.Stream m (MutArray a) -> D.Stream m a
1705+flattenArrays = concat
1706+
1707+{-# INLINE_NORMAL concatRevWith #-}
1708+concatRevWith :: forall m a. (Monad m, Unbox a)
1709+ => (forall b. IO b -> m b) -> D.Stream m (MutArray a) -> D.Stream m a
1710+concatRevWith liftio (D.Stream step state) = D.Stream step' (OuterLoop state)
15131711
15141712 where
15151713
@@ -1527,11 +1725,23 @@ concatRev (D.Stream step state) = D.Stream step' (OuterLoop state)
15271725 return $ D.Skip $ OuterLoop st
15281726
15291727 step' _ (InnerLoop st contents p start) = do
1530− x <- liftIO $ peekAt p contents
1728+ x <- liftio $ peekAt p contents
15311729 let cur = INDEX_PREV(p,a)
15321730 return $ D.Yield x (InnerLoop st contents cur start)
15331731
1732+-- | Use the "readerRev" unfold instead.
1733+--
1734+-- @concat = unfoldMany readerRev@
1735+--
1736+-- We can try this if there are any fusion issues in the unfold.
1737+--
1738+{-# INLINE_NORMAL concatRev #-}
1739+concatRev :: forall m a. (MonadIO m, Unbox a)
1740+ => D.Stream m (MutArray a) -> D.Stream m a
1741+concatRev = concatRevWith liftIO
1742+
15341743{-# DEPRECATED flattenArraysRev "Please use \"unfoldMany readerRev\" instead." #-}
1744+{-# INLINE flattenArraysRev #-}
15351745flattenArraysRev :: forall m a. (MonadIO m, Unbox a)
15361746 => D.Stream m (MutArray a) -> D.Stream m a
15371747flattenArraysRev = concatRev
@@ -1763,7 +1973,7 @@ foldr f z arr = D.foldr f z $ read arr
17631973
17641974-- XXX Keep the bound intact to not lose any free space? Perf impact?
17651975
1766−-- | @writeAppendNUnsafe n arr@ appends up to @n@ input items to the supplied
1976+-- | @unsafeAppendN n arr@ appends up to @n@ input items to the supplied
17671977-- array.
17681978--
17691979-- Unsafe: Do not drive the fold beyond @n@ elements, it will lead to memory
@@ -1772,13 +1982,12 @@ foldr f z arr = D.foldr f z $ read arr
17721982-- Any free space left in the array after appending @n@ elements is lost.
17731983--
17741984-- /Internal/
1775−{-# INLINE_NORMAL writeAppendNUnsafe #-}
1776−writeAppendNUnsafe :: forall m a. (MonadIO m, Unbox a) =>
1985+{-# INLINE_NORMAL unsafeAppendN #-}
1986+unsafeAppendN :: forall m a. (MonadIO m, Unbox a) =>
17771987 Int
17781988 -> m (MutArray a)
17791989 -> Fold m a (MutArray a)
1780−writeAppendNUnsafe n action =
1781− fmap fromArrayUnsafe $ FL.foldlM' step initial
1990+unsafeAppendN n action = fmap fromArrayUnsafe $ FL.foldlM' step initial
17821991
17831992 where
17841993
@@ -1791,7 +2000,7 @@ writeAppendNUnsafe n action =
17912000 -- otherwise we lose that space.
17922001 arr1 <-
17932002 if free < needed
1794− then noinline reallocWith "writeAppendNUnsafeWith" (+ needed) needed arr
2003+ then noinline reallocWith "unsafeAppendN" (+ needed) needed arr
17952004 else return arr
17962005 return $ toArrayUnsafe arr1
17972006
@@ -1799,30 +2008,51 @@ writeAppendNUnsafe n action =
17992008 liftIO $ pokeAt end contents x
18002009 return $ ArrayUnsafe contents start (INDEX_NEXT(end,a))
18012010
2011+{-# DEPRECATED writeAppendNUnsafe "Please use unsafeAppendN instead." #-}
2012+{-# INLINE writeAppendNUnsafe #-}
2013+writeAppendNUnsafe :: forall m a. (MonadIO m, Unbox a) =>
2014+ Int
2015+ -> m (MutArray a)
2016+ -> Fold m a (MutArray a)
2017+writeAppendNUnsafe = unsafeAppendN
2018+
18022019-- | Append @n@ elements to an existing array. Any free space left in the array
18032020-- after appending @n@ elements is lost.
18042021--
1805−-- >>> writeAppendN n initial = Fold.take n (MutArray.writeAppendNUnsafe n initial)
2022+-- >>> appendN n initial = Fold.take n (MutArray.unsafeAppendN n initial)
18062023--
1807−{-# INLINE_NORMAL writeAppendN #-}
2024+{-# INLINE_NORMAL appendN #-}
2025+appendN :: forall m a. (MonadIO m, Unbox a) =>
2026+ Int -> m (MutArray a) -> Fold m a (MutArray a)
2027+appendN n initial = FL.take n (unsafeAppendN n initial)
2028+
2029+-- XXX Deprecate in major
2030+-- {-# DEPRECATED writeAppendN "Please use appendN instead." #-}
2031+{-# INLINE writeAppendN #-}
18082032writeAppendN :: forall m a. (MonadIO m, Unbox a) =>
18092033 Int -> m (MutArray a) -> Fold m a (MutArray a)
1810−writeAppendN n initial = FL.take n (writeAppendNUnsafe n initial)
2034+writeAppendN = appendN
18112035
1812−-- | @writeAppendWith realloc action@ mutates the array generated by @action@ to
2036+-- | @appendWith realloc action@ mutates the array generated by @action@ to
18132037-- append the input stream. If there is no reserved space available in the
18142038-- array it is reallocated to a size in bytes determined by @realloc oldSize@,
18152039-- where @oldSize@ is the current size of the array in bytes.
18162040--
18172041-- Note that the returned array may be a mutated version of original array.
18182042--
1819−-- >>> writeAppendWith sizer = Fold.foldlM' (MutArray.snocWith sizer)
2043+-- >>> appendWith sizer = Fold.foldlM' (MutArray.snocWith sizer)
18202044--
18212045-- /Pre-release/
2046+{-# INLINE appendWith #-}
2047+appendWith :: forall m a. (MonadIO m, Unbox a) =>
2048+ (Int -> Int) -> m (MutArray a) -> Fold m a (MutArray a)
2049+appendWith sizer = FL.foldlM' (snocWith sizer)
2050+
2051+{-# DEPRECATED writeAppendWith "Please use appendWith instead." #-}
18222052{-# INLINE writeAppendWith #-}
18232053writeAppendWith :: forall m a. (MonadIO m, Unbox a) =>
18242054 (Int -> Int) -> m (MutArray a) -> Fold m a (MutArray a)
1825−writeAppendWith sizer = FL.foldlM' (snocWith sizer)
2055+writeAppendWith = appendWith
18262056
18272057-- | @append action@ mutates the array generated by @action@ to append the
18282058-- input stream. If there is no reserved space available in the array it is
@@ -1830,26 +2060,33 @@ writeAppendWith sizer = FL.foldlM' (snocWith sizer)
18302060--
18312061-- Note that the returned array may be a mutated version of original array.
18322062--
1833−-- >>> writeAppend = MutArray.writeAppendWith (* 2)
2063+-- >>> append = MutArray.appendWith (* 2)
18342064--
2065+{-# INLINE append #-}
2066+append :: forall m a. (MonadIO m, Unbox a) =>
2067+ m (MutArray a) -> Fold m a (MutArray a)
2068+append = appendWith (* 2)
2069+
2070+-- XXX Deprecate in major
2071+-- {-# DEPRECATED writeAppend "Please use append instead." #-}
18352072{-# INLINE writeAppend #-}
18362073writeAppend :: forall m a. (MonadIO m, Unbox a) =>
18372074 m (MutArray a) -> Fold m a (MutArray a)
1838−writeAppend = writeAppendWith (* 2)
2075+writeAppend = append
18392076
18402077-- XXX We can carry bound as well in the state to make sure we do not lose the
18412078-- remaining capacity. Need to check perf impact.
18422079--
1843−-- | Like 'writeNUnsafe' but takes a new array allocator @alloc size@ function
1844−-- as argument.
2080+-- | Like 'unsafeCreateOf' but takes a new array allocator @alloc size@
2081+-- function as argument.
18452082--
1846−-- >>> writeNWithUnsafe alloc n = MutArray.writeAppendNUnsafe (alloc n) n
2083+-- >>> unsafeCreateOfWith alloc n = MutArray.unsafeAppendN (alloc n) n
18472084--
18482085-- /Pre-release/
1849−{-# INLINE_NORMAL writeNWithUnsafe #-}
1850−writeNWithUnsafe :: forall m a. (MonadIO m, Unbox a)
2086+{-# INLINE_NORMAL unsafeCreateOfWith #-}
2087+unsafeCreateOfWith :: forall m a. (MonadIO m, Unbox a)
18512088 => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)
1852−writeNWithUnsafe alloc n = fromArrayUnsafe <$> FL.foldlM' step initial
2089+unsafeCreateOfWith alloc n = fromArrayUnsafe <$> FL.foldlM' step initial
18532090
18542091 where
18552092
@@ -1860,40 +2097,64 @@ writeNWithUnsafe alloc n = fromArrayUnsafe <$> FL.foldlM' step initial
18602097 return
18612098 $ ArrayUnsafe contents start (INDEX_NEXT(end,a))
18622099
2100+{-# DEPRECATED writeNWithUnsafe "Please use unsafeCreateOfWith instead." #-}
2101+{-# INLINE writeNWithUnsafe #-}
2102+writeNWithUnsafe :: forall m a. (MonadIO m, Unbox a)
2103+ => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)
2104+writeNWithUnsafe = unsafeCreateOfWith
2105+
18632106{-# INLINE_NORMAL writeNUnsafeAs #-}
18642107writeNUnsafeAs :: forall m a. (MonadIO m, Unbox a)
18652108 => PinnedState -> Int -> Fold m a (MutArray a)
1866−writeNUnsafeAs ps = writeNWithUnsafe (newAs ps)
2109+writeNUnsafeAs ps = unsafeCreateOfWith (newAs ps)
18672110
1868−-- | Like 'writeN' but does not check the array bounds when writing. The fold
2111+-- | Like 'createOf' but does not check the array bounds when writing. The fold
18692112-- driver must not call the step function more than 'n' times otherwise it will
18702113-- corrupt the memory and crash. This function exists mainly because any
18712114-- conditional in the step function blocks fusion causing 10x performance
18722115-- slowdown.
18732116--
1874−-- >>> writeNUnsafe = MutArray.writeNWithUnsafe MutArray.new
2117+-- >>> unsafeCreateOf = MutArray.unsafeCreateOfWith MutArray.emptyOf
18752118--
1876−{-# INLINE_NORMAL writeNUnsafe #-}
2119+{-# INLINE_NORMAL unsafeCreateOf #-}
2120+unsafeCreateOf :: forall m a. (MonadIO m, Unbox a)
2121+ => Int -> Fold m a (MutArray a)
2122+unsafeCreateOf = writeNUnsafeAs Unpinned
2123+
2124+{-# DEPRECATED writeNUnsafe "Please use unsafeCreateOf instead." #-}
2125+{-# INLINE writeNUnsafe #-}
18772126writeNUnsafe :: forall m a. (MonadIO m, Unbox a)
18782127 => Int -> Fold m a (MutArray a)
1879−writeNUnsafe = writeNUnsafeAs Unpinned
2128+writeNUnsafe = unsafeCreateOf
2129+
2130+-- | Like 'unsafeCreateOf' but creates a pinned array.
2131+{-# INLINE_NORMAL unsafePinnedCreateOf #-}
2132+unsafePinnedCreateOf :: forall m a. (MonadIO m, Unbox a)
2133+ => Int -> Fold m a (MutArray a)
2134+unsafePinnedCreateOf = writeNUnsafeAs Pinned
18802135
1881−-- | Like 'writeNUnsafe' but creates a pinned array.
1882−{-# INLINE_NORMAL pinnedWriteNUnsafe #-}
2136+{-# DEPRECATED pinnedWriteNUnsafe "Please use unsafePinnedCreateOf instead." #-}
2137+{-# INLINE pinnedWriteNUnsafe #-}
18832138pinnedWriteNUnsafe :: forall m a. (MonadIO m, Unbox a)
18842139 => Int -> Fold m a (MutArray a)
1885−pinnedWriteNUnsafe = writeNUnsafeAs Pinned
2140+pinnedWriteNUnsafe = unsafePinnedCreateOf
18862141
1887−-- | @writeNWith alloc n@ folds a maximum of @n@ elements into an array
2142+-- | @createOfWith alloc n@ folds a maximum of @n@ elements into an array
18882143-- allocated using the @alloc@ function.
18892144--
1890−-- >>> writeNWith alloc n = Fold.take n (MutArray.writeNWithUnsafe alloc n)
1891−-- >>> writeNWith alloc n = MutArray.writeAppendN (alloc n) n
2145+-- >>> createOfWith alloc n = Fold.take n (MutArray.unsafeCreateOfWith alloc n)
2146+-- >>> createOfWith alloc n = MutArray.appendN (alloc n) n
18922147--
1893−{-# INLINE_NORMAL writeNWith #-}
2148+{-# INLINE_NORMAL createOfWith #-}
2149+createOfWith :: forall m a. (MonadIO m, Unbox a)
2150+ => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)
2151+createOfWith alloc n = FL.take n (unsafeCreateOfWith alloc n)
2152+
2153+-- {-# DEPRECATED writeNWith "Please use createOfWith instead." #-}
2154+{-# INLINE writeNWith #-}
18942155writeNWith :: forall m a. (MonadIO m, Unbox a)
18952156 => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)
1896−writeNWith alloc n = FL.take n (writeNWithUnsafe alloc n)
2157+writeNWith = createOfWith
18972158
18982159{-# INLINE_NORMAL writeNAs #-}
18992160writeNAs ::
@@ -1901,28 +2162,42 @@ writeNAs ::
19012162 => PinnedState
19022163 -> Int
19032164 -> Fold m a (MutArray a)
1904−writeNAs ps = writeNWith (newAs ps)
2165+writeNAs ps = createOfWith (newAs ps)
19052166
1906−-- | @writeN n@ folds a maximum of @n@ elements from the input stream to an
2167+-- | @createOf n@ folds a maximum of @n@ elements from the input stream to an
19072168-- 'MutArray'.
19082169--
1909−-- >>> writeN = MutArray.writeNWith MutArray.new
1910−-- >>> writeN n = Fold.take n (MutArray.writeNUnsafe n)
1911−-- >>> writeN n = MutArray.writeAppendN n (MutArray.new n)
2170+-- >>> createOf = MutArray.createOfWith MutArray.new
2171+-- >>> createOf n = Fold.take n (MutArray.unsafeCreateOf n)
2172+-- >>> createOf n = MutArray.appendN n (MutArray.emptyOf n)
19122173--
1913−{-# INLINE_NORMAL writeN #-}
2174+{-# INLINE_NORMAL createOf #-}
2175+createOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)
2176+createOf = writeNAs Unpinned
2177+
2178+-- XXX Deprecate in major
2179+-- {-# DEPRECATED writeN "Please use createOf instead." #-}
2180+{-# INLINE writeN #-}
19142181writeN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)
1915−writeN = writeNAs Unpinned
2182+writeN = createOf
19162183
1917−-- | Like 'writeN' but creates a pinned array.
1918−{-# INLINE_NORMAL pinnedWriteN #-}
2184+-- | Like 'createOf' but creates a pinned array.
2185+{-# INLINE_NORMAL pinnedCreateOf #-}
2186+pinnedCreateOf ::
2187+ forall m a. (MonadIO m, Unbox a)
2188+ => Int
2189+ -> Fold m a (MutArray a)
2190+pinnedCreateOf = writeNAs Pinned
2191+
2192+{-# DEPRECATED pinnedWriteN "Please use pinnedCreateOf instead." #-}
2193+{-# INLINE pinnedWriteN #-}
19192194pinnedWriteN ::
19202195 forall m a. (MonadIO m, Unbox a)
19212196 => Int
19222197 -> Fold m a (MutArray a)
1923−pinnedWriteN = writeNAs Pinned
2198+pinnedWriteN = pinnedCreateOf
19242199
1925−-- | Like writeNWithUnsafe but writes the array in reverse order.
2200+-- | Like unsafeCreateOfWith but writes the array in reverse order.
19262201--
19272202-- /Internal/
19282203{-# INLINE_NORMAL writeRevNWithUnsafe #-}
@@ -1943,7 +2218,7 @@ writeRevNWithUnsafe alloc n = fromArrayUnsafe <$> FL.foldlM' step initial
19432218 return
19442219 $ ArrayUnsafe contents ptr end
19452220
1946−-- | Like writeNWith but writes the array in reverse order.
2221+-- | Like createOfWith but writes the array in reverse order.
19472222--
19482223-- /Internal/
19492224{-# INLINE_NORMAL writeRevNWith #-}
@@ -1951,31 +2226,36 @@ writeRevNWith :: forall m a. (MonadIO m, Unbox a)
19512226 => (Int -> m (MutArray a)) -> Int -> Fold m a (MutArray a)
19522227writeRevNWith alloc n = FL.take n (writeRevNWithUnsafe alloc n)
19532228
1954−-- | Like writeN but writes the array in reverse order.
2229+-- | Like 'createOf' but writes the array in reverse order.
19552230--
19562231-- /Pre-release/
1957−{-# INLINE_NORMAL writeRevN #-}
2232+{-# INLINE_NORMAL revCreateOf #-}
2233+revCreateOf :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)
2234+revCreateOf = writeRevNWith new
2235+
2236+{-# DEPRECATED writeRevN "Please use revCreateOf instead." #-}
2237+{-# INLINE writeRevN #-}
19582238writeRevN :: forall m a. (MonadIO m, Unbox a) => Int -> Fold m a (MutArray a)
1959−writeRevN = writeRevNWith new
2239+writeRevN = revCreateOf
19602240
19612241-- | @pinnedWriteNAligned align n@ folds a maximum of @n@ elements from the
19622242-- input stream to a 'MutArray' aligned to the given size.
19632243--
1964−-- >>> pinnedWriteNAligned align = MutArray.writeNWith (MutArray.pinnedNewAligned align)
1965−-- >>> pinnedWriteNAligned align n = MutArray.writeAppendN n (MutArray.pinnedNewAligned align n)
2244+-- >>> pinnedWriteNAligned align = MutArray.createOfWith (MutArray.pinnedNewAligned align)
2245+-- >>> pinnedWriteNAligned align n = MutArray.appendN n (MutArray.pinnedNewAligned align n)
19662246--
19672247-- /Pre-release/
19682248--
19692249{-# INLINE_NORMAL pinnedWriteNAligned #-}
19702250pinnedWriteNAligned :: forall m a. (MonadIO m, Unbox a)
19712251 => Int -> Int -> Fold m a (MutArray a)
1972−pinnedWriteNAligned align = writeNWith (pinnedNewAligned align)
2252+pinnedWriteNAligned align = createOfWith (pinnedNewAligned align)
19732253
19742254-- XXX Buffer to a list instead?
19752255
19762256-- | Buffer a stream into a stream of arrays.
19772257--
1978−-- >>> buildChunks n = Fold.many (MutArray.writeN n) Fold.toStreamK
2258+-- >>> buildChunks n = Fold.many (MutArray.createOf n) Fold.toStreamK
19792259--
19802260-- Breaking an array into an array stream can be useful to consume a large
19812261-- array sequentially such that memory of the array is released incrementatlly.
@@ -1987,9 +2267,10 @@ pinnedWriteNAligned align = writeNWith (pinnedNewAligned align)
19872267{-# INLINE_NORMAL buildChunks #-}
19882268buildChunks :: (MonadIO m, Unbox a) =>
19892269 Int -> Fold m a (StreamK n (MutArray a))
1990−buildChunks n = FL.many (writeN n) FL.toStreamK
2270+buildChunks n = FL.many (createOf n) FL.toStreamK
19912271
19922272{-# DEPRECATED writeChunks "Please use buildChunks instead." #-}
2273+{-# INLINE writeChunks #-}
19932274writeChunks :: (MonadIO m, Unbox a) =>
19942275 Int -> Fold m a (StreamK n (MutArray a))
19952276writeChunks = buildChunks
@@ -1997,29 +2278,29 @@ writeChunks = buildChunks
19972278{-# INLINE_NORMAL writeWithAs #-}
19982279writeWithAs :: forall m a. (MonadIO m, Unbox a)
19992280 => PinnedState -> Int -> Fold m a (MutArray a)
2000−-- writeWithAs ps n = FL.rmapM rightSize $ writeAppendWith (* 2) (newAs ps n)
2281+-- writeWithAs ps n = FL.rmapM rightSize $ appendWith (* 2) (newAs ps n)
20012282writeWithAs ps elemCount =
20022283 FL.rmapM extract $ FL.foldlM' step initial
20032284
20042285 where
20052286
20062287 initial = do
2007− when (elemCount < 0) $ error "writeWith: elemCount is negative"
2288+ when (elemCount < 0) $ error "createWith: elemCount is negative"
20082289 newAs ps elemCount
20092290
20102291 step arr@(MutArray _ start end bound) x
20112292 | INDEX_NEXT(end,a) > bound = do
20122293 let oldSize = end - start
20132294 newSize = max (oldSize * 2) 1
2014− arr1 <- liftIO $ reallocExplicit (SIZE_OF(a)) newSize arr
2295+ arr1 <- liftIO $ reallocExplicitAs ps (SIZE_OF(a)) newSize arr
20152296 snocUnsafe arr1 x
20162297 step arr x = snocUnsafe arr x
20172298
20182299 extract = liftIO . rightSize
20192300
2020−-- XXX Compare writeWith with fromStreamD which uses an array of streams
2301+-- XXX Compare createWith with fromStreamD which uses an array of streams
20212302-- implementation. We can write this using buildChunks above if that is faster.
2022−-- If writeWith is faster then we should use that to implement
2303+-- If createWith is faster then we should use that to implement
20232304-- fromStreamD.
20242305--
20252306-- XXX The realloc based implementation needs to make one extra copy if we use
@@ -2034,39 +2315,56 @@ writeWithAs ps elemCount =
20342315-- XXX check if GHC's memory allocator is efficient enough. We can try the C
20352316-- malloc to compare against.
20362317
2037−-- | @writeWith minCount@ folds the whole input to a single array. The array
2318+-- | @createWith minCount@ folds the whole input to a single array. The array
20382319-- starts at a size big enough to hold minCount elements, the size is doubled
20392320-- every time the array needs to be grown.
20402321--
20412322-- /Caution! Do not use this on infinite streams./
20422323--
2043−-- >>> f n = MutArray.writeAppendWith (* 2) (MutArray.new n)
2044−-- >>> writeWith n = Fold.rmapM MutArray.rightSize (f n)
2045−-- >>> writeWith n = Fold.rmapM MutArray.fromChunksK (MutArray.buildChunks n)
2324+-- >>> f n = MutArray.appendWith (* 2) (MutArray.emptyOf n)
2325+-- >>> createWith n = Fold.rmapM MutArray.rightSize (f n)
2326+-- >>> createWith n = Fold.rmapM MutArray.fromChunksK (MutArray.buildChunks n)
20462327--
20472328-- /Pre-release/
2048−{-# INLINE_NORMAL writeWith #-}
2329+{-# INLINE_NORMAL createWith #-}
2330+createWith :: forall m a. (MonadIO m, Unbox a)
2331+ => Int -> Fold m a (MutArray a)
2332+-- createWith n = FL.rmapM rightSize $ appendWith (* 2) (emptyOf n)
2333+createWith = writeWithAs Unpinned
2334+
2335+{-# DEPRECATED writeWith "Please use createWith instead." #-}
2336+{-# INLINE writeWith #-}
20492337writeWith :: forall m a. (MonadIO m, Unbox a)
20502338 => Int -> Fold m a (MutArray a)
2051−-- writeWith n = FL.rmapM rightSize $ writeAppendWith (* 2) (new n)
2052−writeWith = writeWithAs Unpinned
2339+writeWith = createWith
20532340
20542341-- | Fold the whole input to a single array.
20552342--
2056−-- Same as 'writeWith' using an initial array size of 'arrayChunkBytes' bytes
2343+-- Same as 'createWith' using an initial array size of 'arrayChunkBytes' bytes
20572344-- rounded up to the element size.
20582345--
20592346-- /Caution! Do not use this on infinite streams./
20602347--
2348+{-# INLINE create #-}
2349+create :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)
2350+create = createWith (allocBytesToElemCount (undefined :: a) arrayChunkBytes)
2351+
2352+-- XXX Deprecate in major
2353+-- {-# DEPRECATED write "Please use create instead." #-}
20612354{-# INLINE write #-}
20622355write :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)
2063−write = writeWith (allocBytesToElemCount (undefined :: a) arrayChunkBytes)
2356+write = create
20642357
2065−-- | Like 'write' but creates a pinned array.
2358+-- | Like 'create' but creates a pinned array.
2359+{-# INLINE pinnedCreate #-}
2360+pinnedCreate :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)
2361+pinnedCreate =
2362+ writeWithAs Pinned (allocBytesToElemCount (undefined :: a) arrayChunkBytes)
2363+
2364+{-# DEPRECATED pinnedWrite "Please use pinnedCreate instead." #-}
20662365{-# INLINE pinnedWrite #-}
20672366pinnedWrite :: forall m a. (MonadIO m, Unbox a) => Fold m a (MutArray a)
2068−pinnedWrite =
2069− writeWithAs Pinned (allocBytesToElemCount (undefined :: a) arrayChunkBytes)
2367+pinnedWrite = pinnedCreate
20702368
20712369-------------------------------------------------------------------------------
20722370-- construct from streams, known size
@@ -2086,17 +2384,18 @@ fromStreamDNAs ps limit str = do
20862384 liftIO $ pokeAt ptr arrContents x
20872385 return $ INDEX_NEXT(ptr,a)
20882386
2089−-- | Use the 'writeN' fold instead.
2387+-- | Use the 'createOf' fold instead.
20902388--
2091−-- >>> fromStreamN n = Stream.fold (MutArray.writeN n)
2389+-- >>> fromStreamN n = Stream.fold (MutArray.createOf n)
20922390--
20932391{-# INLINE_NORMAL fromStreamN #-}
20942392fromStreamN :: forall m a. (MonadIO m, Unbox a)
20952393 => Int -> D.Stream m a -> m (MutArray a)
2096−-- fromStreamDN n = D.fold (writeN n)
2394+-- fromStreamDN n = D.fold (createOf n)
20972395fromStreamN = fromStreamDNAs Unpinned
20982396
20992397{-# DEPRECATED fromStreamDN "Please use fromStreamN instead." #-}
2398+{-# INLINE fromStreamDN #-}
21002399fromStreamDN :: forall m a. (MonadIO m, Unbox a)
21012400 => Int -> D.Stream m a -> m (MutArray a)
21022401fromStreamDN = fromStreamN
@@ -2119,20 +2418,20 @@ pinnedFromListN n xs = fromStreamDNAs Pinned n $ D.fromList xs
21192418-- /Pre-release/
21202419{-# INLINE fromListRevN #-}
21212420fromListRevN :: (MonadIO m, Unbox a) => Int -> [a] -> m (MutArray a)
2122−fromListRevN n xs = D.fold (writeRevN n) $ D.fromList xs
2421+fromListRevN n xs = D.fold (revCreateOf n) $ D.fromList xs
21232422
21242423-- | Convert a pure stream in Identity monad to a mutable array.
21252424{-# INLINABLE fromPureStreamN #-}
21262425fromPureStreamN :: (MonadIO m, Unbox a) =>
21272426 Int -> Stream Identity a -> m (MutArray a)
21282427fromPureStreamN n xs =
2129− D.fold (writeN n) $ D.morphInner (return . runIdentity) xs
2428+ D.fold (createOf n) $ D.morphInner (return . runIdentity) xs
21302429
21312430-- | Convert a pure stream in Identity monad to a mutable array.
21322431{-# INLINABLE fromPureStream #-}
21332432fromPureStream :: (MonadIO m, Unbox a) => Stream Identity a -> m (MutArray a)
21342433fromPureStream xs =
2135− D.fold write $ D.morphInner (return . runIdentity) xs
2434+ D.fold create $ D.morphInner (return . runIdentity) xs
21362435
21372436{-# INLINABLE fromPtrN #-}
21382437fromPtrN :: MonadIO m => Int -> Ptr Word8 -> m (MutArray Word8)
@@ -2141,7 +2440,7 @@ fromPtrN len addr = do
21412440 -- XXX We can implement a stream copy in a similar way by streaming Word64
21422441 -- first and then remaining Word8.
21432442 arr <- new len
2144− _ <- asUnpinnedPtrUnsafe arr
2443+ _ <- unsafeAsPtr arr
21452444 (\ptr -> liftIO $ c_memcpy ptr addr (fromIntegral len))
21462445 return (arr {arrEnd = len})
21472446
@@ -2157,7 +2456,7 @@ fromByteStr# addr = do
21572456 len <- liftIO $ c_strlen (Ptr addr)
21582457 let lenInt = fromIntegral len
21592458 arr <- new lenInt
2160− _ <- asUnpinnedPtrUnsafe arr (\ptr -> liftIO $ c_memcpy ptr (Ptr addr) len)
2459+ _ <- unsafeAsPtr arr (\ptr -> liftIO $ c_memcpy ptr (Ptr addr) len)
21612460 return (arr {arrEnd = lenInt})
21622461
21632462-------------------------------------------------------------------------------
@@ -2216,6 +2515,7 @@ fromChunksK :: (Unbox a, MonadIO m) =>
22162515fromChunksK = fromChunkskAs Unpinned
22172516
22182517{-# DEPRECATED fromArrayStreamK "Please use fromChunksK instead." #-}
2518+{-# INLINE fromArrayStreamK #-}
22192519fromArrayStreamK :: (Unbox a, MonadIO m) =>
22202520 StreamK m (MutArray a) -> m (MutArray a)
22212521fromArrayStreamK = fromChunksK
@@ -2227,7 +2527,7 @@ fromStreamDAs ps m =
22272527 arrayStreamKFromStreamDAs Unpinned m >>= fromChunkskAs ps
22282528
22292529-- | Create an 'Array' from a stream. This is useful when we want to create a
2230−-- single array from a stream of unknown size. 'writeN' is at least twice
2530+-- single array from a stream of unknown size. 'createOf' is at least twice
22312531-- as efficient when the size is already known.
22322532--
22332533-- Note that if the input stream is too large memory allocation for the array
@@ -2239,19 +2539,19 @@ fromStreamDAs ps m =
22392539fromStream :: (MonadIO m, Unbox a) => Stream m a -> m (MutArray a)
22402540fromStream = fromStreamDAs Unpinned
22412541
2242−-- fromStream (Stream m) = P.fold write m
2542+-- fromStream (Stream m) = P.fold create m
22432543-- CAUTION: a very large number (millions) of arrays can degrade performance
22442544-- due to GC overhead because we need to buffer the arrays before we flatten
22452545-- all the arrays.
22462546--
2247−-- XXX Compare if this is faster or "fold write".
2547+-- XXX Compare if this is faster or "fold create".
22482548--
22492549-- | We could take the approach of doubling the memory allocation on each
22502550-- overflow. This would result in more or less the same amount of copying as in
22512551-- the chunking approach. However, if we have to shrink in the end then it may
22522552-- result in an extra copy of the entire data.
22532553--
2254−-- >>> fromStreamD = StreamD.fold MutArray.write
2554+-- >>> fromStreamD = StreamD.fold MutArray.create
22552555--
22562556{-# INLINE fromStreamD #-}
22572557{-# DEPRECATED fromStreamD "Please use fromStream instead." #-}
@@ -2281,6 +2581,10 @@ fromListRev xs = fromListRevN (Prelude.length xs) xs
22812581-- Cloning
22822582-------------------------------------------------------------------------------
22832583
2584+-- Arrays are aligned on 64-bit boundaries. The fastest way to copy an array is
2585+-- to unsafeCast it to Word64, read it, write it to Word64 array and unsafeCast
2586+-- it again. We can use SIMD read/write as well.
2587+
22842588{-# INLINE cloneAs #-}
22852589cloneAs ::
22862590 ( MonadIO m
@@ -2372,7 +2676,7 @@ spliceWith :: forall m a. (MonadIO m, Unbox a) =>
23722676 (Int -> Int -> Int) -> MutArray a -> MutArray a -> m (MutArray a)
23732677spliceWith sizer dst@(MutArray _ start end bound) src = do
23742678{-
2375− let f = writeAppendWith (`sizer` byteLength src) (return dst)
2679+ let f = appendWith (`sizer` byteLength src) (return dst)
23762680 in D.fold f (toStreamD src)
23772681-}
23782682 assert (end <= bound) (return ())
@@ -2440,7 +2744,7 @@ splitOn predicate arr =
24402744{-# INLINE breakOn #-}
24412745breakOn :: MonadIO m
24422746 => Word8 -> MutArray Word8 -> m (MutArray Word8, Maybe (MutArray Word8))
2443−breakOn sep arr@MutArray{..} = asUnpinnedPtrUnsafe arr $ \p -> liftIO $ do
2747+breakOn sep arr@MutArray{..} = unsafeAsPtr arr $ \p -> liftIO $ do
24442748 -- XXX We do not need memchr here, we can use a Haskell equivalent.
24452749 -- Need efficient stream based primitives that work on Word64.
24462750 loc <- c_memchr p sep (fromIntegral $ byteLength arr)
@@ -2463,33 +2767,41 @@ breakOn sep arr@MutArray{..} = asUnpinnedPtrUnsafe arr $ \p -> liftIO $ do
24632767 }
24642768 )
24652769
2770+-- | Like 'splitAt' but does not check whether the index is valid.
2771+--
2772+{-# INLINE unsafeSplitAt #-}
2773+unsafeSplitAt :: forall a. Unbox a =>
2774+ Int -> MutArray a -> (MutArray a, MutArray a)
2775+unsafeSplitAt i MutArray{..} =
2776+ let off = i * SIZE_OF(a)
2777+ p = arrStart + off
2778+ in ( MutArray
2779+ { arrContents = arrContents
2780+ , arrStart = arrStart
2781+ , arrEnd = p
2782+ , arrBound = p
2783+ }
2784+ , MutArray
2785+ { arrContents = arrContents
2786+ , arrStart = p
2787+ , arrEnd = arrEnd
2788+ , arrBound = arrBound
2789+ }
2790+ )
2791+
24662792-- | Create two slices of an array without copying the original array. The
24672793-- specified index @i@ is the first index of the second slice.
24682794--
24692795{-# INLINE splitAt #-}
24702796splitAt :: forall a. Unbox a => Int -> MutArray a -> (MutArray a, MutArray a)
2471−splitAt i arr@MutArray{..} =
2797+splitAt i arr =
24722798 let maxIndex = length arr - 1
24732799 in if i < 0
24742800 then error "sliceAt: negative array index"
24752801 else if i > maxIndex
24762802 then error $ "sliceAt: specified array index " ++ show i
24772803 ++ " is beyond the maximum index " ++ show maxIndex
2478− else let off = i * SIZE_OF(a)
2479− p = arrStart + off
2480− in ( MutArray
2481− { arrContents = arrContents
2482− , arrStart = arrStart
2483− , arrEnd = p
2484− , arrBound = p
2485− }
2486− , MutArray
2487− { arrContents = arrContents
2488− , arrStart = p
2489− , arrEnd = arrEnd
2490− , arrBound = arrBound
2491− }
2492− )
2804+ else unsafeSplitAt i arr
24932805
24942806-------------------------------------------------------------------------------
24952807-- Casting
@@ -2539,9 +2851,13 @@ cast arr =
25392851-- guarantees that unsafe calls will be performed in the calling thread. Making
25402852-- it safe to pass heap-allocated objects to unsafe functions.
25412853
2542−-- Should we just name it asPtr, the unsafety is implicit for any pointer
2854+-- XXX Should we just name it asPtr, the unsafety is implicit for any pointer
25432855-- operations. And we are safe from Haskell perspective because we will be
25442856-- pinning the memory.
2857+--
2858+-- XXX we cannot pass the length of the ptr here as in some cases it may not be
2859+-- available e.g. a null terminated C string. However, we can create another
2860+-- flavor of the API e.g. asPtrN.
25452861
25462862-- | Use a @MutArray a@ as @Ptr a@. This is useful when we want to pass an
25472863-- array as a pointer to some operating system call or to a "safe" FFI call.
@@ -2559,16 +2875,21 @@ cast arr =
25592875--
25602876-- /Pre-release/
25612877--
2878+{-# INLINE unsafePinnedAsPtr #-}
2879+unsafePinnedAsPtr :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b
2880+unsafePinnedAsPtr arr f =
2881+ Unboxed.unsafePinnedAsPtr
2882+ (arrContents arr) (\ptr -> f (ptr `plusPtr` arrStart arr))
2883+
2884+{-# DEPRECATED asPtrUnsafe "Please use unsafePinnedAsPtr instead." #-}
25622885{-# INLINE asPtrUnsafe #-}
25632886asPtrUnsafe :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b
2564−asPtrUnsafe arr f =
2565− Unboxed.asPtrUnsafe
2566− (arrContents arr) (\ptr -> f (ptr `plusPtr` arrStart arr))
2887+asPtrUnsafe = unsafePinnedAsPtr
25672888
2568−{-# INLINE asUnpinnedPtrUnsafe #-}
2569−asUnpinnedPtrUnsafe :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b
2570−asUnpinnedPtrUnsafe arr f =
2571− Unboxed.asUnpinnedPtrUnsafe
2889+{-# INLINE unsafeAsPtr #-}
2890+unsafeAsPtr :: MonadIO m => MutArray a -> (Ptr a -> m b) -> m b
2891+unsafeAsPtr arr f =
2892+ Unboxed.unsafeAsPtr
25722893 (arrContents arr) (\ptr -> f (ptr `plusPtr` arrStart arr))
25732894
25742895-------------------------------------------------------------------------------
@@ -2625,46 +2946,48 @@ byteEq arr1 arr2 = fmap (EQ ==) $ byteCmp arr1 arr2
26252946-- Compact
26262947-------------------------------------------------------------------------------
26272948
2628−-- | Parser @pCompactLE n@ coalesces adjacent arrays in the input stream
2629−-- only if the combined size would be less than or equal to n.
2630−--
2631−-- /Internal/
2632−{-# INLINE_NORMAL pCompactLE #-}
2633−pCompactLE ::
2949+-- Note: LE versions avoid an extra copy compared to GE. LE parser trades
2950+-- backtracking one array in lieu of avoiding a copy. However, LE and GE both
2951+-- can leave some memory unused. They can split the last array to fit it
2952+-- exactly in the space.
2953+
2954+{-# INLINE_NORMAL pCompactLeAs #-}
2955+pCompactLeAs ::
26342956 forall m a. (MonadIO m, Unbox a)
2635− => Int -> Parser (MutArray a) m (MutArray a)
2636−pCompactLE n = Parser step initial extract
2957+ => PinnedState -> Int -> Parser (MutArray a) m (MutArray a)
2958+pCompactLeAs ps maxElems = Parser step initial extract
26372959
26382960 where
26392961
2640− nBytes = n * SIZE_OF(a)
2962+ maxBytes = maxElems * SIZE_OF(a)
26412963
26422964 functionName = "Streamly.Internal.Data.MutArray.pCompactLE"
26432965
26442966 initial =
26452967 return
2646− $ if n <= 0
2968+ $ if maxElems <= 0
26472969 then error
26482970 $ functionName
26492971 ++ ": the size of arrays ["
2650− ++ show n ++ "] must be a natural number"
2972+ ++ show maxElems ++ "] must be a natural number"
26512973 else Parser.IPartial Nothing
26522974
26532975 step Nothing arr =
26542976 return
26552977 $ let len = byteLength arr
2656− in if len >= nBytes
2978+ in if len >= maxBytes
26572979 then Parser.Done 0 arr
26582980 else Parser.Partial 0 (Just arr)
2981+ -- XXX Split the last array to use the space more compactly.
26592982 step (Just buf) arr =
26602983 let len = byteLength buf + byteLength arr
2661− in if len > nBytes
2984+ in if len > maxBytes
26622985 then return $ Parser.Done 1 buf
26632986 else do
2664− -- XXX check this in the Nothing case
26652987 buf1 <-
2666− if byteCapacity buf < nBytes
2667− then liftIO $ realloc nBytes buf
2988+ if byteCapacity buf < maxBytes
2989+ then liftIO $ reallocExplicitAs
2990+ ps (SIZE_OF(a)) maxBytes buf
26682991 else return buf
26692992 buf2 <- spliceUnsafe buf1 arr
26702993 return $ Parser.Partial 0 (Just buf2)
@@ -2672,6 +2995,30 @@ pCompactLE n = Parser step initial extract
26722995 extract Nothing = return $ Parser.Done 0 nil
26732996 extract (Just buf) = return $ Parser.Done 0 buf
26742997
2998+-- | Parser @pCompactLE maxElems@ coalesces adjacent arrays in the input stream
2999+-- only if the combined size would be less than or equal to @maxElems@
3000+-- elements. Note that it won't split an array if the original array is already
3001+-- larger than maxElems.
3002+--
3003+-- @maxElems@ must be greater than 0.
3004+--
3005+-- Generates unpinned arrays irrespective of the pinning status of input
3006+-- arrays.
3007+--
3008+-- /Internal/
3009+{-# INLINE pCompactLE #-}
3010+pCompactLE ::
3011+ forall m a. (MonadIO m, Unbox a)
3012+ => Int -> Parser (MutArray a) m (MutArray a)
3013+pCompactLE = pCompactLeAs Unpinned
3014+
3015+-- | Pinned version of 'pCompactLE'.
3016+{-# INLINE pPinnedCompactLE #-}
3017+pPinnedCompactLE ::
3018+ forall m a. (MonadIO m, Unbox a)
3019+ => Int -> Parser (MutArray a) m (MutArray a)
3020+pPinnedCompactLE = pCompactLeAs Pinned
3021+
26753022data SpliceState s arr
26763023 = SpliceInitial s
26773024 | SpliceBuffering s arr
@@ -2683,48 +3030,47 @@ data SpliceState s arr
26833030-- immutable array never has additional space so a new array is allocated
26843031-- instead of mutating it.
26853032
2686−-- | Scan @rCompactLE n@ coalesces adjacent arrays in the input stream
2687−-- only if the combined size would be less than or equal to n.
2688−--
2689−-- /Internal/
2690−{-# INLINE_NORMAL rCompactLE #-}
2691−rCompactLE :: (MonadIO m, Unbox a)
2692− => Int -> D.Stream m (MutArray a) -> D.Stream m (MutArray a)
2693−rCompactLE n (D.Stream step state) =
3033+{-# INLINE_NORMAL compactLeAs #-}
3034+compactLeAs :: forall m a. (MonadIO m, Unbox a)
3035+ => PinnedState -> Int -> D.Stream m (MutArray a) -> D.Stream m (MutArray a)
3036+compactLeAs ps maxElems (D.Stream step state) =
26943037 D.Stream step' (SpliceInitial state)
26953038
26963039 where
26973040
3041+ maxBytes = maxElems * SIZE_OF(a)
3042+
26983043 functionName = "Streamly.Internal.Data.MutArray.rCompactLE"
26993044
27003045 {-# INLINE_LATE step' #-}
27013046 step' gst (SpliceInitial st) = do
2702− when (n <= 0) $
3047+ when (maxElems <= 0) $
27033048 -- XXX we can pass the module string from the higher level API
2704− error $ functionName ++ ": the size of arrays [" ++ show n
3049+ error $ functionName ++ ": the size of arrays [" ++ show maxElems
27053050 ++ "] must be a natural number"
27063051 r <- step gst st
27073052 case r of
27083053 D.Yield arr s -> return $
27093054 let len = byteLength arr
2710− in if len >= n
3055+ in if len >= maxBytes
27113056 then D.Skip (SpliceYielding arr (SpliceInitial s))
27123057 else D.Skip (SpliceBuffering s arr)
27133058 D.Skip s -> return $ D.Skip (SpliceInitial s)
27143059 D.Stop -> return D.Stop
27153060
3061+ -- XXX Split the last array to use the space more compactly.
27163062 step' gst (SpliceBuffering st buf) = do
27173063 r <- step gst st
27183064 case r of
27193065 D.Yield arr s -> do
27203066 let len = byteLength buf + byteLength arr
2721− if len > n
3067+ if len > maxBytes
27223068 then return $
27233069 D.Skip (SpliceYielding buf (SpliceBuffering s arr))
27243070 else do
2725− -- XXX check this in the SpliceInitial case
2726− buf1 <- if byteCapacity buf < n
2727− then liftIO $ realloc n buf
3071+ buf1 <- if byteCapacity buf < maxBytes
3072+ then liftIO $ reallocExplicitAs
3073+ ps (SIZE_OF(a)) maxBytes buf
27283074 else return buf
27293075 buf2 <- spliceUnsafe buf1 arr
27303076 return $ D.Skip (SpliceBuffering s buf2)
@@ -2735,108 +3081,119 @@ rCompactLE n (D.Stream step state) =
27353081
27363082 step' _ (SpliceYielding arr next) = return $ D.Yield arr next
27373083
2738−-- | Fold @fCompactGE n@ coalesces adjacent arrays in the input stream
2739−-- until the size becomes greater than or equal to n.
2740−--
2741−{-# INLINE_NORMAL fCompactGE #-}
2742−fCompactGE ::
3084+
3085+{-# INLINE_NORMAL fCompactGeAs #-}
3086+fCompactGeAs ::
27433087 forall m a. (MonadIO m, Unbox a)
2744− => Int -> FL.Fold m (MutArray a) (MutArray a)
2745−fCompactGE n = Fold step initial extract extract
3088+ => PinnedState -> Int -> FL.Fold m (MutArray a) (MutArray a)
3089+fCompactGeAs ps minElems = Fold step initial extract extract
27463090
27473091 where
27483092
2749− nBytes = n * SIZE_OF(a)
3093+ minBytes = minElems * SIZE_OF(a)
27503094
27513095 functionName = "Streamly.Internal.Data.MutArray.fCompactGE"
27523096
27533097 initial =
27543098 return
2755− $ if n < 0
3099+ $ if minElems < 0
27563100 then error
27573101 $ functionName
27583102 ++ ": the size of arrays ["
2759− ++ show n ++ "] must be a natural number"
3103+ ++ show minElems ++ "] must be a natural number"
27603104 else FL.Partial Nothing
27613105
27623106 step Nothing arr =
27633107 return
27643108 $ let len = byteLength arr
2765− in if len >= nBytes
3109+ in if len >= minBytes
27663110 then FL.Done arr
27673111 else FL.Partial (Just arr)
3112+ -- XXX Buffer arrays as a list to avoid copy and reallocations
27683113 step (Just buf) arr = do
27693114 let len = byteLength buf + byteLength arr
27703115 buf1 <-
27713116 if byteCapacity buf < len
2772− then liftIO $ realloc (max len nBytes) buf
3117+ then liftIO $ reallocExplicitAs
3118+ ps (SIZE_OF(a)) (max minBytes len) buf
27733119 else return buf
27743120 buf2 <- spliceUnsafe buf1 arr
2775− if len >= n
3121+ if len >= minBytes
27763122 then return $ FL.Done buf2
27773123 else return $ FL.Partial (Just buf2)
27783124
27793125 extract Nothing = return nil
27803126 extract (Just buf) = return buf
27813127
2782−-- | Like 'compactGE' but for transforming folds instead of stream.
2783−--
2784−-- >>> lCompactGE n = Fold.many (MutArray.fCompactGE n)
3128+-- | Fold @fCompactGE minElems@ coalesces adjacent arrays in the input stream
3129+-- until the size becomes greater than or equal to @minElems@.
27853130--
2786−{-# INLINE_NORMAL lCompactGE #-}
2787−lCompactGE :: (MonadIO m, Unbox a)
2788− => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()
2789−-- lCompactGE n = Fold.many (fCompactGE n)
2790−lCompactGE n (Fold step1 initial1 _ final1) =
3131+-- Generates unpinned arrays irrespective of the pinning status of input
3132+-- arrays.
3133+{-# INLINE fCompactGE #-}
3134+fCompactGE ::
3135+ forall m a. (MonadIO m, Unbox a)
3136+ => Int -> FL.Fold m (MutArray a) (MutArray a)
3137+fCompactGE = fCompactGeAs Unpinned
3138+
3139+-- | Pinned version of 'fCompactGE'.
3140+{-# INLINE fPinnedCompactGE #-}
3141+fPinnedCompactGE ::
3142+ forall m a. (MonadIO m, Unbox a)
3143+ => Int -> FL.Fold m (MutArray a) (MutArray a)
3144+fPinnedCompactGE = fCompactGeAs Pinned
3145+
3146+{-# INLINE_NORMAL lCompactGeAs #-}
3147+lCompactGeAs :: forall m a. (MonadIO m, Unbox a)
3148+ => PinnedState -> Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()
3149+-- The fold version turns out to be a little bit slower.
3150+-- lCompactGeAs ps n = FL.many (fCompactGeAs ps n)
3151+lCompactGeAs ps minElems (Fold step1 initial1 _ final1) =
27913152 Fold step initial extract final
27923153
27933154 where
27943155
3156+ minBytes = minElems * SIZE_OF(a)
3157+
27953158 functionName = "Streamly.Internal.Data.MutArray.lCompactGE"
27963159
27973160 initial = do
2798− when (n <= 0) $
3161+ when (minElems <= 0) $
27993162 -- XXX we can pass the module string from the higher level API
28003163 error $ functionName ++ ": the size of arrays ["
2801− ++ show n ++ "] must be a natural number"
3164+ ++ show minElems ++ "] must be a natural number"
28023165
28033166 r <- initial1
28043167 return $ first (Tuple' Nothing) r
28053168
2806− step (Tuple' Nothing r1) arr =
2807− let len = byteLength arr
2808− in if len >= n
2809− then do
2810− r <- step1 r1 arr
2811− case r of
2812− FL.Done _ -> return $ FL.Done ()
2813− FL.Partial s -> do
2814− _ <- final1 s
2815− res <- initial1
2816− return $ first (Tuple' Nothing) res
2817− else return $ FL.Partial $ Tuple' (Just arr) r1
2818−
2819− step (Tuple' (Just buf) r1) arr = do
2820− let len = byteLength buf + byteLength arr
2821− buf1 <- if byteCapacity buf < len
2822− then liftIO $ realloc (max n len) buf
2823− else return buf
2824− buf2 <- spliceUnsafe buf1 arr
2825−
2826− -- XXX this is common in both the equations of step
2827− if len >= n
3169+ {-# INLINE runInner #-}
3170+ runInner len acc buf =
3171+ if len >= minBytes
28283172 then do
2829− r <- step1 r1 buf2
3173+ r <- step1 acc buf
28303174 case r of
28313175 FL.Done _ -> return $ FL.Done ()
28323176 FL.Partial s -> do
28333177 _ <- final1 s
28343178 res <- initial1
28353179 return $ first (Tuple' Nothing) res
2836− else return $ FL.Partial $ Tuple' (Just buf2) r1
3180+ else return $ FL.Partial $ Tuple' (Just buf) acc
3181+
3182+ step (Tuple' Nothing r1) arr =
3183+ runInner (byteLength arr) r1 arr
3184+
3185+ -- XXX Buffer arrays as a list to avoid copy and reallocations
3186+ step (Tuple' (Just buf) r1) arr = do
3187+ let len = byteLength buf + byteLength arr
3188+ buf1 <- if byteCapacity buf < len
3189+ then liftIO $ reallocExplicitAs
3190+ ps (SIZE_OF(a)) (max minBytes len) buf
3191+ else return buf
3192+ buf2 <- spliceUnsafe buf1 arr
3193+ runInner len r1 buf2
28373194
28383195 -- XXX Several folds do extract >=> final, therefore, we need to make final
2839− -- return "m b" rather than using extract post it if we want extract to be
3196+ -- return "m b" rather than using extract post it if we want extract to be
28403197 -- partial.
28413198 --
28423199 -- extract forces the pending buffer to be sent to the fold which is not
@@ -2850,6 +3207,23 @@ lCompactGE n (Fold step1 initial1 _ final1) =
28503207 FL.Partial rr -> final1 rr
28513208 FL.Done _ -> return ()
28523209
3210+-- | Like 'compactGE' but for transforming folds instead of stream.
3211+--
3212+-- >>> lCompactGE n = Fold.many (MutArray.fCompactGE n)
3213+--
3214+-- Generates unpinned arrays irrespective of the pinning status of input
3215+-- arrays.
3216+{-# INLINE lCompactGE #-}
3217+lCompactGE :: forall m a. (MonadIO m, Unbox a)
3218+ => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()
3219+lCompactGE = lCompactGeAs Unpinned
3220+
3221+-- | Pinned version of 'lCompactGE'.
3222+{-# INLINE lPinnedCompactGE #-}
3223+lPinnedCompactGE :: forall m a. (MonadIO m, Unbox a)
3224+ => Int -> Fold m (MutArray a) () -> Fold m (MutArray a) ()
3225+lPinnedCompactGE = lCompactGeAs Pinned
3226+
28533227-- | @compactGE n stream@ coalesces adjacent arrays in the @stream@ until
28543228-- the size becomes greater than or equal to @n@.
28553229--