@@ -6,6 +6,10 @@
66-- Stability : experimental
77-- Portability : GHC
88--
9+-- Mutable arrays and file system files are quite similar, they can grow and
10+-- their content is mutable. Therefore, both have similar APIs as well. We
11+-- strive to keep the API consistent for both. Ideally, you should be able to
12+-- replace one with another with little changes to the code.
913
1014module Streamly.Internal.Data.Array.Foreign.Mut.Type
1115 (
@@ -15,11 +19,11 @@ module Streamly.Internal.Data.Array.Foreign.Mut.Type
1519
1620 -- * Construction
1721 , fromForeignPtrUnsafe
18− , unsafeWithNewArray
22+ , withNewArrayUnsafe
1923 , newArray
2024 , newArrayAligned
2125 , newArrayAlignedUnmanaged
22− , newArrayAlignedAllocWith
26+ , newArrayWith
2327
2428 -- * From containers
2529 , fromList
@@ -65,14 +69,14 @@ module Streamly.Internal.Data.Array.Foreign.Mut.Type
6569 , append
6670
6771 , writeWith
68− , writeNAllocWith
72+ , writeNWithUnsafe
73+ , writeNWith
6974 , writeN
7075 , writeNUnsafe
7176 , ArrayUnsafe (..)
7277 , writeNAligned
7378 , writeNAlignedUnmanaged
7479 , write
75− , writeAligned
7680
7781 -- * Unfolds
7882 , ReadUState
@@ -278,12 +282,27 @@ fromForeignPtrUnsafe fp end bound =
278282-- Construction
279283-------------------------------------------------------------------------------
280284
281−-- | allocate a new empty array using the provided allocator function.
282−{-# INLINE newArrayAlignedAllocWith #-}
283−newArrayAlignedAllocWith :: forall m a. (MonadIO m, Storable a)
285+-- XXX Change the names to use "new" instead of "newArray". That way we can use
286+-- the same names for managed file system objects as well. For unmanaged ones
287+-- we can use open/create etc as usual.
288+
289+-- GHC always guarantees word-aligned memory, alignment is important only when
290+-- we need more than that. See stg_newAlignedPinnedByteArrayzh and
291+-- allocatePinned in GHC source.
292+
293+-- | @newArrayWith allocator alignment count@ allocates a new array of zero
294+-- length and with a capacity to hold @count@ elements, using @allocator
295+-- size alignment@ as the memory allocator function.
296+--
297+-- Alignment must be greater than or equal to machine word size and a power of
298+-- 2.
299+--
300+-- /Pre-release/
301+{-# INLINE newArrayWith #-}
302+newArrayWith :: forall m a. (MonadIO m, Storable a)
284303 => (Int -> Int -> m (ForeignPtr a)) -> Int -> Int -> m (Array a)
285−newArrayAlignedAllocWith alloc alignSize count = do
286− let size = count * sizeOf (undefined :: a)
304+newArrayWith alloc alignSize count = do
305+ let size = max (count * sizeOf (undefined :: a)) 0
287306 fptr <- alloc size alignSize
288307 let p = unsafeForeignPtrToPtr fptr
289308 return $ Array
@@ -292,23 +311,29 @@ newArrayAlignedAllocWith alloc alignSize count = do
292311 , aBound = p `plusPtr` size
293312 }
294313
295−-- | Allocate a new empty array using the specified alignment and using
296−-- unmanaged pinned memory. The memory will not be automatically freed by GHC.
297−-- This could be useful in allocate-once global data structures. Use carefully
298−-- as incorrect use can lead to memory leak.
314+-- | Like 'newArrayWith' but using an allocator that allocates unmanaged pinned
315+-- memory. The memory will never be freed by GHC. This could be useful in
316+-- allocate-once global data structures. Use carefully as incorrect use can
317+-- lead to memory leak.
318+--
319+-- /Internal/
299320{-# INLINE newArrayAlignedUnmanaged #-}
300−newArrayAlignedUnmanaged :: (MonadIO m, Storable a) => Int -> Int -> m (Array a)
301−newArrayAlignedUnmanaged =
302− newArrayAlignedAllocWith mallocForeignPtrAlignedUnmanagedBytes
321+newArrayAlignedUnmanaged :: (MonadIO m, Storable a) =>
322+ Int -> Int -> m (Array a)
323+newArrayAlignedUnmanaged = newArrayWith mallocForeignPtrAlignedUnmanagedBytes
303324
304325 where
305326
306327 mallocForeignPtrAlignedUnmanagedBytes size alignSize =
307328 liftIO $ Malloc.mallocForeignPtrAlignedUnmanagedBytes size alignSize
308329
330+-- | Like 'newArrayWith' but using an allocator that aligns the memory to the
331+-- alignment dictated by the 'Storable' instance of the type.
332+--
333+-- /Internal/
309334{-# INLINE newArrayAligned #-}
310335newArrayAligned :: (MonadIO m, Storable a) => Int -> Int -> m (Array a)
311−newArrayAligned = newArrayAlignedAllocWith mallocForeignPtrAlignedBytes
336+newArrayAligned = newArrayWith mallocForeignPtrAlignedBytes
312337
313338 where
314339
@@ -317,19 +342,23 @@ newArrayAligned = newArrayAlignedAllocWith mallocForeignPtrAlignedBytes
317342
318343-- XXX can unaligned allocation be more efficient when alignment is not needed?
319344--
320−-- | Allocate an empty array that can hold 'count' items. The memory of the
321−-- array is uninitialized.
345+-- | Allocates an empty array that can hold 'count' items. The memory of the
346+-- array is uninitialized and the allocation is aligned as per the 'Storable'
347+-- instance of the type.
322348--
349+-- /Pre-release/
323350{-# INLINE newArray #-}
324351newArray :: forall m a. (MonadIO m, Storable a) => Int -> m (Array a)
325352newArray = newArrayAligned (alignment (undefined :: a))
326353
327354-- | Allocate an Array of the given size and run an IO action passing the array
328355-- start pointer.
329−{-# INLINE unsafeWithNewArray #-}
330−unsafeWithNewArray ::
356+--
357+-- /Internal/
358+{-# INLINE withNewArrayUnsafe #-}
359+withNewArrayUnsafe ::
331360 (MonadIO m, Storable a) => Int -> (Ptr a -> m ()) -> m (Array a)
332−unsafeWithNewArray count f = do
361+withNewArrayUnsafe count f = do
333362 arr <- newArray count
334363 unsafeWithForeignPtrM (aStart arr) $ \p -> f p >> return arr
335364
@@ -1079,55 +1108,72 @@ append :: forall m a. (MonadIO m, Storable a) =>
10791108 m (Array a) -> Fold m a (Array a)
10801109append = appendWith (* 2)
10811110
1082−{-# INLINE_NORMAL writeNAllocWith #-}
1083−writeNAllocWith :: forall m a. (MonadIO m, Storable a)
1111+-- | @writeNWith alloc n@ folds a maximum of @n@ elements into an array
1112+-- allocated using the @alloc@ function.
1113+--
1114+-- >>> writeNWith alloc n = Fold.take n (Array.writeNWithUnsafe alloc n)
1115+-- >>> writeNWith alloc n = Array.appendN (alloc n) n
1116+--
1117+{-# INLINE_NORMAL writeNWith #-}
1118+writeNWith :: forall m a. (MonadIO m, Storable a)
10841119 => (Int -> m (Array a)) -> Int -> Fold m a (Array a)
1085−writeNAllocWith alloc n = FL.take n (writeNUnsafeWith alloc n)
1120+writeNWith alloc n = FL.take n (writeNWithUnsafe alloc n)
10861121
10871122-- | @writeN n@ folds a maximum of @n@ elements from the input stream to an
10881123-- 'Array'.
10891124--
1125+-- >>> writeN = Array.writeNWith Array.newArray
10901126-- >>> writeN n = Fold.take n (Array.writeNUnsafe n)
10911127-- >>> writeN n = Array.appendN (Array.newArray n) n
10921128--
10931129-- @since 0.7.0
10941130{-# INLINE_NORMAL writeN #-}
10951131writeN :: forall m a. (MonadIO m, Storable a) => Int -> Fold m a (Array a)
1096−writeN = writeNAllocWith newArray
1132+writeN = writeNWith newArray
10971133
1098−-- | @writeNAligned alignment n@ folds a maximum of @n@ elements from the input
1134+-- | @writeNAligned align n@ folds a maximum of @n@ elements from the input
10991135-- stream to an 'Array' aligned to the given size.
11001136--
1137+-- >>> writeNAligned align = Array.writeNWith (Array.newArrayAligned align)
1138+-- >>> writeNAligned align n = Array.appendN (Array.newArrayAligned align n) n
1139+--
11011140-- /Pre-release/
11021141--
11031142{-# INLINE_NORMAL writeNAligned #-}
11041143writeNAligned :: forall m a. (MonadIO m, Storable a)
11051144 => Int -> Int -> Fold m a (Array a)
1106−writeNAligned alignSize = writeNAllocWith (newArrayAligned alignSize)
1145+writeNAligned align = writeNWith (newArrayAligned align)
11071146
1108−-- | @writeNAlignedUnmanaged n@ folds a maximum of @n@ elements from the input
1109−-- stream to an 'Array' aligned to the given size and using unmanaged memory.
1110−-- This could be useful to allocate memory that we need to allocate only once
1111−-- in the lifetime of the program.
1147+-- | @writeNAlignedUnmanaged align n@ folds a maximum of @n@ elements from the
1148+-- input stream to an 'Array' whose starting address is aligned to @align@
1149+-- bytes and is allocated using unmanaged memory (never freed). This could be
1150+-- useful to allocate memory that we need to allocate only once in the lifetime
1151+-- of the program.
1152+--
1153+-- >>> f = Array.newArrayAlignedUnmanaged
1154+-- >>> writeNAlignedUnmanaged a = Array.writeNWith (f a)
1155+-- >>> writeNAlignedUnmanaged a n = Array.appendN (f a n) n
11121156--
11131157-- /Pre-release/
11141158--
11151159{-# INLINE_NORMAL writeNAlignedUnmanaged #-}
11161160writeNAlignedUnmanaged :: forall m a. (MonadIO m, Storable a)
11171161 => Int -> Int -> Fold m a (Array a)
1118−writeNAlignedUnmanaged alignSize =
1119− writeNAllocWith (newArrayAlignedUnmanaged alignSize)
1162+writeNAlignedUnmanaged align = writeNWith (newArrayAlignedUnmanaged align)
11201163
11211164-- XXX We can carry bound as well in the state to make sure we do not lose the
11221165-- remaining capacity. Need to check perf impact.
11231166--
1124−-- | This API is only useful when we do not want to use the remaining space in
1125−-- the array to append anything else later as that space is lost.
1167+-- | Like 'writeNUnsafe' but takes a new array allocator @alloc size@ function
1168+-- as argument.
1169+--
1170+-- >>> writeNWithUnsafe alloc n = Array.appendNUnsafe (alloc n) n
11261171--
1127−{-# INLINE_NORMAL writeNUnsafeWith #-}
1128−writeNUnsafeWith :: forall m a. (MonadIO m, Storable a)
1172+-- /Pre-release/
1173+{-# INLINE_NORMAL writeNWithUnsafe #-}
1174+writeNWithUnsafe :: forall m a. (MonadIO m, Storable a)
11291175 => (Int -> m (Array a)) -> Int -> Fold m a (Array a)
1130−writeNUnsafeWith alloc n = Fold step initial extract
1176+writeNWithUnsafe alloc n = Fold step initial extract
11311177
11321178 where
11331179
@@ -1149,11 +1195,13 @@ writeNUnsafeWith alloc n = Fold step initial extract
11491195-- conditional in the step function blocks fusion causing 10x performance
11501196-- slowdown.
11511197--
1198+-- >>> writeNUnsafe = Array.writeNWithUnsafe Array.newArray
1199+--
11521200-- @since 0.7.0
11531201{-# INLINE_NORMAL writeNUnsafe #-}
11541202writeNUnsafe :: forall m a. (MonadIO m, Storable a)
11551203 => Int -> Fold m a (Array a)
1156−writeNUnsafe = writeNUnsafeWith newArray
1204+writeNUnsafe = writeNWithUnsafe newArray
11571205
11581206-- XXX Buffer to a list instead?
11591207--
@@ -1190,6 +1238,13 @@ writeChunks n = FL.many (writeN n) FL.toStreamK
11901238-- XXX check if GHC's memory allocator is efficient enough. We can try the C
11911239-- malloc to compare against.
11921240
1241+-- | @writeWith minCount@ folds the whole input to a single array. The array
1242+-- starts at a size big enough to hold minCount elements, the size is doubled
1243+-- every time the array needs to be grown.
1244+--
1245+-- /Caution! Do not use this on infinite streams./
1246+--
1247+-- /Pre-release/
11931248{-# INLINE_NORMAL writeWith #-}
11941249writeWith :: forall m a. (MonadIO m, Storable a)
11951250 => Int -> Int -> Fold m a (Array a)
@@ -1221,6 +1276,9 @@ writeWith alignSize elemCount =
12211276
12221277-- | Fold the whole input to a single array.
12231278--
1279+-- Same as 'writeWith' using an initial array size of 'arrayChunkSize' bytes
1280+-- rounded up to the element size.
1281+--
12241282-- /Caution! Do not use this on infinite streams./
12251283--
12261284-- @since 0.7.0
@@ -1230,21 +1288,6 @@ write = writeWith (alignment (undefined :: a))
12301288 (bytesToElemCount (undefined :: a)
12311289 (mkChunkSize 1024))
12321290
1233−-- | Like 'write' but the array memory is aligned according to the specified
1234−-- alignment size. This could be useful when we have specific alignment, for
1235−-- example, cache aligned arrays for lookup table etc.
1236−--
1237−-- /Caution! Do not use this on infinite streams./
1238−--
1239−-- @since 0.7.0
1240−{-# INLINE writeAligned #-}
1241−writeAligned :: forall m a. (MonadIO m, Storable a)
1242− => Int -> Fold m a (Array a)
1243−writeAligned alignSize =
1244− writeWith alignSize
1245− (bytesToElemCount (undefined :: a)
1246− (mkChunkSize 1024))
1247−
12481291-------------------------------------------------------------------------------
12491292-- construct from streams, known size
12501293-------------------------------------------------------------------------------