@@ -6,6 +6,10 @@
6-- Stability : experimental
6-- Stability : experimental
7-- Portability : GHC
7-- Portability : GHC
9+-- Mutable arrays and file system files are quite similar, they can grow and
10+-- their content is mutable. Therefore, both have similar APIs as well. We
11+-- strive to keep the API consistent for both. Ideally, you should be able to
12+-- replace one with another with little changes to the code.
10module Streamly.Internal.Data.Array.Foreign.Mut.Type
14module Streamly.Internal.Data.Array.Foreign.Mut.Type
@@ -15,11 +19,11 @@ module Streamly.Internal.Data.Array.Foreign.Mut.Type
16 -- * Construction
20 -- * Construction
17 , fromForeignPtrUnsafe
21 , fromForeignPtrUnsafe
18− , unsafeWithNewArray
22+ , withNewArrayUnsafe
19 , newArray
23 , newArray
20 , newArrayAligned
24 , newArrayAligned
21 , newArrayAlignedUnmanaged
25 , newArrayAlignedUnmanaged
22− , newArrayAlignedAllocWith
26+ , newArrayWith
24 -- * From containers
28 -- * From containers
25 , fromList
29 , fromList
@@ -65,14 +69,14 @@ module Streamly.Internal.Data.Array.Foreign.Mut.Type
67 , writeWith
71 , writeWith
68− , writeNAllocWith
72+ , writeNWithUnsafe
70 , writeNUnsafe
75 , writeNUnsafe
71 , ArrayUnsafe (..)
76 , ArrayUnsafe (..)
72 , writeNAligned
77 , writeNAligned
73 , writeNAlignedUnmanaged
78 , writeNAlignedUnmanaged
77 -- * Unfolds
81 -- * Unfolds
78 , ReadUState
82 , ReadUState
@@ -278,12 +282,27 @@ fromForeignPtrUnsafe fp end bound =
278-- Construction
282-- Construction
279-------------------------------------------------------------------------------
283-------------------------------------------------------------------------------
285+-- XXX Change the names to use "new" instead of "newArray". That way we can use
286+-- the same names for managed file system objects as well. For unmanaged ones
287+-- we can use open/create etc as usual.
289+-- GHC always guarantees word-aligned memory, alignment is important only when
290+-- we need more than that. See stg_newAlignedPinnedByteArrayzh and
291+-- allocatePinned in GHC source.
293+-- | @newArrayWith allocator alignment count@ allocates a new array of zero
294+-- length and with a capacity to hold @count@ elements, using @allocator
281−-- | allocate a new empty array using the provided allocator function.
295+-- size alignment@ as the memory allocator function.
282−{-# INLINE newArrayAlignedAllocWith #-}
296+--
297+-- Alignment must be greater than or equal to machine word size and a power of
301+{-# INLINE newArrayWith #-}
283−newArrayAlignedAllocWith :: forall m a. (MonadIO m, Storable a)
302+newArrayWith :: forall m a. (MonadIO m, Storable a)
284 => (Int -> Int -> m (ForeignPtr a)) -> Int -> Int -> m (Array a)
303 => (Int -> Int -> m (ForeignPtr a)) -> Int -> Int -> m (Array a)
285−newArrayAlignedAllocWith alloc alignSize count = do
304+newArrayWith alloc alignSize count = do
286− let size = count * sizeOf (undefined :: a)
305+ let size = max (count * sizeOf (undefined :: a)) 0
287 fptr <- alloc size alignSize
306 fptr <- alloc size alignSize
288 let p = unsafeForeignPtrToPtr fptr
307 let p = unsafeForeignPtrToPtr fptr
289 return $ Array
308 return $ Array
@@ -292,23 +311,29 @@ newArrayAlignedAllocWith alloc alignSize count = do
292 , aBound = p `plusPtr` size
311 , aBound = p `plusPtr` size
295−-- | Allocate a new empty array using the specified alignment and using
314+-- | Like 'newArrayWith' but using an allocator that allocates unmanaged pinned
296−-- unmanaged pinned memory. The memory will not be automatically freed by GHC.
315+-- memory. The memory will never be freed by GHC. This could be useful in
297−-- This could be useful in allocate-once global data structures. Use carefully
316+-- allocate-once global data structures. Use carefully as incorrect use can
298−-- as incorrect use can lead to memory leak.
317+-- lead to memory leak.
299{-# INLINE newArrayAlignedUnmanaged #-}
320{-# INLINE newArrayAlignedUnmanaged #-}
300−newArrayAlignedUnmanaged :: (MonadIO m, Storable a) => Int -> Int -> m (Array a)
321+newArrayAlignedUnmanaged :: (MonadIO m, Storable a) =>
301−newArrayAlignedUnmanaged =
322+ Int -> Int -> m (Array a)
302− newArrayAlignedAllocWith mallocForeignPtrAlignedUnmanagedBytes
323+newArrayAlignedUnmanaged = newArrayWith mallocForeignPtrAlignedUnmanagedBytes
306 mallocForeignPtrAlignedUnmanagedBytes size alignSize =
327 mallocForeignPtrAlignedUnmanagedBytes size alignSize =
307 liftIO $ Malloc.mallocForeignPtrAlignedUnmanagedBytes size alignSize
328 liftIO $ Malloc.mallocForeignPtrAlignedUnmanagedBytes size alignSize
330+-- | Like 'newArrayWith' but using an allocator that aligns the memory to the
331+-- alignment dictated by the 'Storable' instance of the type.
309{-# INLINE newArrayAligned #-}
334{-# INLINE newArrayAligned #-}
310newArrayAligned :: (MonadIO m, Storable a) => Int -> Int -> m (Array a)
335newArrayAligned :: (MonadIO m, Storable a) => Int -> Int -> m (Array a)
311−newArrayAligned = newArrayAlignedAllocWith mallocForeignPtrAlignedBytes
336+newArrayAligned = newArrayWith mallocForeignPtrAlignedBytes
@@ -317,19 +342,23 @@ newArrayAligned = newArrayAlignedAllocWith mallocForeignPtrAlignedBytes
318-- XXX can unaligned allocation be more efficient when alignment is not needed?
343-- XXX can unaligned allocation be more efficient when alignment is not needed?
320−-- | Allocate an empty array that can hold 'count' items. The memory of the
345+-- | Allocates an empty array that can hold 'count' items. The memory of the
321−-- array is uninitialized.
346+-- array is uninitialized and the allocation is aligned as per the 'Storable'
347+-- instance of the type.
323{-# INLINE newArray #-}
350{-# INLINE newArray #-}
324newArray :: forall m a. (MonadIO m, Storable a) => Int -> m (Array a)
351newArray :: forall m a. (MonadIO m, Storable a) => Int -> m (Array a)
325newArray = newArrayAligned (alignment (undefined :: a))
352newArray = newArrayAligned (alignment (undefined :: a))
327-- | Allocate an Array of the given size and run an IO action passing the array
354-- | Allocate an Array of the given size and run an IO action passing the array
328-- start pointer.
355-- start pointer.
329−{-# INLINE unsafeWithNewArray #-}
356+--
330−unsafeWithNewArray ::
357+-- /Internal/
358+{-# INLINE withNewArrayUnsafe #-}
359+withNewArrayUnsafe ::
331 (MonadIO m, Storable a) => Int -> (Ptr a -> m ()) -> m (Array a)
360 (MonadIO m, Storable a) => Int -> (Ptr a -> m ()) -> m (Array a)
332−unsafeWithNewArray count f = do
361+withNewArrayUnsafe count f = do
333 arr <- newArray count
362 arr <- newArray count
334 unsafeWithForeignPtrM (aStart arr) $ \p -> f p >> return arr
363 unsafeWithForeignPtrM (aStart arr) $ \p -> f p >> return arr
@@ -1079,55 +1108,72 @@ append :: forall m a. (MonadIO m, Storable a) =>
1079 m (Array a) -> Fold m a (Array a)
1108 m (Array a) -> Fold m a (Array a)
1080append = appendWith (* 2)
1109append = appendWith (* 2)
1111+-- | @writeNWith alloc n@ folds a maximum of @n@ elements into an array
1112+-- allocated using the @alloc@ function.
1114+-- >>> writeNWith alloc n = Fold.take n (Array.writeNWithUnsafe alloc n)
1115+-- >>> writeNWith alloc n = Array.appendN (alloc n) n
1082−{-# INLINE_NORMAL writeNAllocWith #-}
1117+{-# INLINE_NORMAL writeNWith #-}
1083−writeNAllocWith :: forall m a. (MonadIO m, Storable a)
1118+writeNWith :: forall m a. (MonadIO m, Storable a)
1084 => (Int -> m (Array a)) -> Int -> Fold m a (Array a)
1119 => (Int -> m (Array a)) -> Int -> Fold m a (Array a)
1085−writeNAllocWith alloc n = FL.take n (writeNUnsafeWith alloc n)
1120+writeNWith alloc n = FL.take n (writeNWithUnsafe alloc n)
1087-- | @writeN n@ folds a maximum of @n@ elements from the input stream to an
1122-- | @writeN n@ folds a maximum of @n@ elements from the input stream to an
1088-- 'Array'.
1123-- 'Array'.
1125+-- >>> writeN = Array.writeNWith Array.newArray
1090-- >>> writeN n = Fold.take n (Array.writeNUnsafe n)
1126-- >>> writeN n = Fold.take n (Array.writeNUnsafe n)
1091-- >>> writeN n = Array.appendN (Array.newArray n) n
1127-- >>> writeN n = Array.appendN (Array.newArray n) n
1093-- @since 0.7.0
1129-- @since 0.7.0
1094{-# INLINE_NORMAL writeN #-}
1130{-# INLINE_NORMAL writeN #-}
1095writeN :: forall m a. (MonadIO m, Storable a) => Int -> Fold m a (Array a)
1131writeN :: forall m a. (MonadIO m, Storable a) => Int -> Fold m a (Array a)
1096−writeN = writeNAllocWith newArray
1132+writeN = writeNWith newArray
1098−-- | @writeNAligned alignment n@ folds a maximum of @n@ elements from the input
1134+-- | @writeNAligned align n@ folds a maximum of @n@ elements from the input
1099-- stream to an 'Array' aligned to the given size.
1135-- stream to an 'Array' aligned to the given size.
1137+-- >>> writeNAligned align = Array.writeNWith (Array.newArrayAligned align)
1138+-- >>> writeNAligned align n = Array.appendN (Array.newArrayAligned align n) n
1101-- /Pre-release/
1140-- /Pre-release/
1103{-# INLINE_NORMAL writeNAligned #-}
1142{-# INLINE_NORMAL writeNAligned #-}
1104writeNAligned :: forall m a. (MonadIO m, Storable a)
1143writeNAligned :: forall m a. (MonadIO m, Storable a)
1105 => Int -> Int -> Fold m a (Array a)
1144 => Int -> Int -> Fold m a (Array a)
1106−writeNAligned alignSize = writeNAllocWith (newArrayAligned alignSize)
1145+writeNAligned align = writeNWith (newArrayAligned align)
1108−-- | @writeNAlignedUnmanaged n@ folds a maximum of @n@ elements from the input
1147+-- | @writeNAlignedUnmanaged align n@ folds a maximum of @n@ elements from the
1109−-- stream to an 'Array' aligned to the given size and using unmanaged memory.
1148+-- input stream to an 'Array' whose starting address is aligned to @align@
1149+-- bytes and is allocated using unmanaged memory (never freed). This could be
1110−-- This could be useful to allocate memory that we need to allocate only once
1150+-- useful to allocate memory that we need to allocate only once in the lifetime
1111−-- in the lifetime of the program.
1151+-- of the program.
1153+-- >>> f = Array.newArrayAlignedUnmanaged
1154+-- >>> writeNAlignedUnmanaged a = Array.writeNWith (f a)
1155+-- >>> writeNAlignedUnmanaged a n = Array.appendN (f a n) n
1113-- /Pre-release/
1157-- /Pre-release/
1115{-# INLINE_NORMAL writeNAlignedUnmanaged #-}
1159{-# INLINE_NORMAL writeNAlignedUnmanaged #-}
1116writeNAlignedUnmanaged :: forall m a. (MonadIO m, Storable a)
1160writeNAlignedUnmanaged :: forall m a. (MonadIO m, Storable a)
1117 => Int -> Int -> Fold m a (Array a)
1161 => Int -> Int -> Fold m a (Array a)
1118−writeNAlignedUnmanaged alignSize =
1162+writeNAlignedUnmanaged align = writeNWith (newArrayAlignedUnmanaged align)
1119− writeNAllocWith (newArrayAlignedUnmanaged alignSize)
1121-- XXX We can carry bound as well in the state to make sure we do not lose the
1164-- XXX We can carry bound as well in the state to make sure we do not lose the
1122-- remaining capacity. Need to check perf impact.
1165-- remaining capacity. Need to check perf impact.
1124−-- | This API is only useful when we do not want to use the remaining space in
1167+-- | Like 'writeNUnsafe' but takes a new array allocator @alloc size@ function
1125−-- the array to append anything else later as that space is lost.
1168+-- as argument.
1170+-- >>> writeNWithUnsafe alloc n = Array.appendNUnsafe (alloc n) n
1127−{-# INLINE_NORMAL writeNUnsafeWith #-}
1173+{-# INLINE_NORMAL writeNWithUnsafe #-}
1128−writeNUnsafeWith :: forall m a. (MonadIO m, Storable a)
1174+writeNWithUnsafe :: forall m a. (MonadIO m, Storable a)
1129 => (Int -> m (Array a)) -> Int -> Fold m a (Array a)
1175 => (Int -> m (Array a)) -> Int -> Fold m a (Array a)
1130−writeNUnsafeWith alloc n = Fold step initial extract
1176+writeNWithUnsafe alloc n = Fold step initial extract
@@ -1149,11 +1195,13 @@ writeNUnsafeWith alloc n = Fold step initial extract
1149-- conditional in the step function blocks fusion causing 10x performance
1195-- conditional in the step function blocks fusion causing 10x performance
1150-- slowdown.
1196-- slowdown.
1198+-- >>> writeNUnsafe = Array.writeNWithUnsafe Array.newArray
1152-- @since 0.7.0
1200-- @since 0.7.0
1153{-# INLINE_NORMAL writeNUnsafe #-}
1201{-# INLINE_NORMAL writeNUnsafe #-}
1154writeNUnsafe :: forall m a. (MonadIO m, Storable a)
1202writeNUnsafe :: forall m a. (MonadIO m, Storable a)
1155 => Int -> Fold m a (Array a)
1203 => Int -> Fold m a (Array a)
1156−writeNUnsafe = writeNUnsafeWith newArray
1204+writeNUnsafe = writeNWithUnsafe newArray
1158-- XXX Buffer to a list instead?
1206-- XXX Buffer to a list instead?
@@ -1190,6 +1238,13 @@ writeChunks n = FL.many (writeN n) FL.toStreamK
1190-- XXX check if GHC's memory allocator is efficient enough. We can try the C
1238-- XXX check if GHC's memory allocator is efficient enough. We can try the C
1191-- malloc to compare against.
1239-- malloc to compare against.
1241+-- | @writeWith minCount@ folds the whole input to a single array. The array
1242+-- starts at a size big enough to hold minCount elements, the size is doubled
1243+-- every time the array needs to be grown.
1245+-- /Caution! Do not use this on infinite streams./
1193{-# INLINE_NORMAL writeWith #-}
1248{-# INLINE_NORMAL writeWith #-}
1194writeWith :: forall m a. (MonadIO m, Storable a)
1249writeWith :: forall m a. (MonadIO m, Storable a)
1195 => Int -> Int -> Fold m a (Array a)
1250 => Int -> Int -> Fold m a (Array a)
@@ -1221,6 +1276,9 @@ writeWith alignSize elemCount =
1222-- | Fold the whole input to a single array.
1277-- | Fold the whole input to a single array.
1279+-- Same as 'writeWith' using an initial array size of 'arrayChunkSize' bytes
1280+-- rounded up to the element size.
1224-- /Caution! Do not use this on infinite streams./
1282-- /Caution! Do not use this on infinite streams./
1226-- @since 0.7.0
1284-- @since 0.7.0
@@ -1230,21 +1288,6 @@ write = writeWith (alignment (undefined :: a))
1230 (bytesToElemCount (undefined :: a)
1288 (bytesToElemCount (undefined :: a)
1231 (mkChunkSize 1024))
1289 (mkChunkSize 1024))
1233−-- | Like 'write' but the array memory is aligned according to the specified
1234−-- alignment size. This could be useful when we have specific alignment, for
1235−-- example, cache aligned arrays for lookup table etc.
1237−-- /Caution! Do not use this on infinite streams./
1240−{-# INLINE writeAligned #-}
1241−writeAligned :: forall m a. (MonadIO m, Storable a)
1242− => Int -> Fold m a (Array a)
1243−writeAligned alignSize =
1244− writeWith alignSize
1245− (bytesToElemCount (undefined :: a)
1246− (mkChunkSize 1024))
1248-------------------------------------------------------------------------------
1291-------------------------------------------------------------------------------
1249-- construct from streams, known size
1292-- construct from streams, known size
1250-------------------------------------------------------------------------------
1293-------------------------------------------------------------------------------