@@ -17,46 +17,128 @@ module Streamly.Internal.Data.Array.Foreign.Mut.Type
17 -- $arrayNotes
17 -- $arrayNotes
18 Array (..)
18 Array (..)
20+ -- * Constructing and Writing
20− -- * Construction
21+ -- ** Construction
21− , fromForeignPtrUnsafe
22+ -- , nil
22− , withNewArrayUnsafe
23+
24+ -- *** Uninitialized Arrays
23 , newArray
25 , newArray
24 , newArrayAligned
26 , newArrayAligned
25 , newArrayAlignedUnmanaged
27 , newArrayAlignedUnmanaged
26 , newArrayWith
28 , newArrayWith
30+ -- *** Initialized Arrays
40+ , writeNAlignedUnmanaged
28− -- * From containers
48+ -- ** From containers
29− , fromList
49+ , fromForeignPtrUnsafe
30 , fromListN
50 , fromListN
31 , fromStreamDN
52 , fromStreamDN
32 , fromStreamD
53 , fromStreamD
34− -- * Resizing
55+ -- * Random writes
60+ -- , putFrom -- start writing at the given position
61+ -- , putUpto -- write from beginning up to the given position
71+ -- * Growing and Shrinking
72+ -- Arrays grow only at the end, though it is possible to grow on both sides
73+ -- and therefore have a cons as well as snoc. But that will require two
74+ -- bounds in the array representation.
76+ -- ** Appending elements
83+ -- ** Appending streams
90+ -- These are not the same as slicing the array at the beginning, they may
91+ -- reduce the length as well as the capacity of the array.
96+ -- * Eliminating and Reading
120+ -- * Memory Management
35 , realloc
123 , realloc
36 , rightSize
126 , rightSize
38 -- * Size
128 -- * Size
40 , byteLength
130 , byteLength
41 , byteCapacity
132 , byteCapacity
42 , bytesFree
133 , bytesFree
50− -- * Mutation
135+ -- * In-place Mutation Algorithms
51− , putIndexUnsafe
136+ , reverse
52− , putIndex
137+ , permute
53− , modifyIndexUnsafe
138+ , partitionBy
54− , modifyIndex
139+ , shuffleBy
55− , snocUnsafe
140+ , divideBy
56− , snocWith
141+ , mergeBy
61 -- * Casting
143 -- * Casting
@@ -64,61 +146,45 @@ module Streamly.Internal.Data.Array.Foreign.Mut.Type
64 , asBytes
146 , asBytes
65 , asPtrUnsafe
147 , asPtrUnsafe
71 -- * Folding
149 -- * Folding
76− -- * Composable Folds
154+ -- * Arrays of arrays
77− -- XXX These should be Refolds
155+ -- We can add dimensionality parameter to the array type to get
78− , appendNUnsafe
156+ -- multidimensional arrays. Multidimensional arrays would just be a
79− , appendN
157+ -- convenience wrapper on top of single dimensional arrays.
83− , writeWith
159+ -- | Operations dealing with multiple arrays, streams of arrays or
84− , writeNWithUnsafe
160+ -- multidimensional array representations.
90− , writeNAlignedUnmanaged
93− -- * Unfolds
162+ -- ** Construct from streams
94− , ReadUState
163+ , arraysOf
95− , read
164+ , arrayStreamKFromStreamD
96− , readRev
165+ , writeChunks
167+ -- ** Eliminate to streams
98 , flattenArrays
168 , flattenArrays
99 , flattenArraysRev
169 , flattenArraysRev
101− -- * To containers
172+ -- ** Construct from arrays
102− , toStreamD
173+ -- get chunks without copying
103− , toStreamDRev
174+ , getSliceUnsafe
104− , toStreamK
175+ , getSlice
105− , toStreamKRev
176+ -- , getSlicesFromLenN
106− , toList
177+ , splitAt -- XXX should be able to express using getSlice
108− -- * Combining
180+ -- ** Appending arrays
109 , spliceWith
182 , spliceWith
111 , spliceExp
184 , spliceExp
112− , spliceCopy
185+ -- , putSlice
113−
186+ -- , appendSlice
114− -- * Splitting
187+ -- , appendSliceFrom
118− -- * Stream of arrays
123 -- * Utilities
189 -- * Utilities
124 , bytesToElemCount
190 , bytesToElemCount
@@ -135,7 +201,7 @@ import Control.DeepSeq (NFData(..))
135import Control.Monad (when, void)
201import Control.Monad (when, void)
136import Control.Monad.IO.Class (MonadIO(..))
202import Control.Monad.IO.Class (MonadIO(..))
137import Data.Bits ((.&.))
203import Data.Bits ((.&.))
138−import Data.Functor.Identity (runIdentity)
204+import Data.Functor.Identity (Identity(runIdentity))
139#if __GLASGOW_HASKELL__ < 808
205#if __GLASGOW_HASKELL__ < 808
140import Data.Semigroup (Semigroup(..))
206import Data.Semigroup (Semigroup(..))
@@ -172,7 +238,8 @@ import qualified Streamly.Internal.Data.Stream.StreamD.Type as D
172import qualified Streamly.Internal.Data.Stream.StreamK.Type as K
238import qualified Streamly.Internal.Data.Stream.StreamK.Type as K
173import qualified Streamly.Internal.Foreign.Malloc as Malloc
239import qualified Streamly.Internal.Foreign.Malloc as Malloc
241+import Prelude hiding
175−import Prelude hiding (length, foldr, read, unlines, splitAt)
242+ (length, foldr, read, unlines, splitAt, reverse, truncate)
177#if MIN_VERSION_base(4,10,0)
244#if MIN_VERSION_base(4,10,0)
178import Foreign.ForeignPtr (plusForeignPtr)
245import Foreign.ForeignPtr (plusForeignPtr)
@@ -205,7 +272,8 @@ foreign import ccall unsafe "string.h memcmp" c_memcmp
205 :: Ptr Word8 -> Ptr Word8 -> CSize -> IO CInt
272 :: Ptr Word8 -> Ptr Word8 -> CSize -> IO CInt
207-- | Given a 'Storable' type (unused first arg) and a number of bytes, return
274-- | Given a 'Storable' type (unused first arg) and a number of bytes, return
208−-- how many elements of that type are required to hold those bytes.
275+-- how many elements of that type will fit in after rounding the bytes up to
209{-# INLINE bytesToElemCount #-}
277{-# INLINE bytesToElemCount #-}
210bytesToElemCount :: Storable a => a -> Int -> Int
278bytesToElemCount :: Storable a => a -> Int -> Int
211bytesToElemCount x n =
279bytesToElemCount x n =
@@ -298,6 +366,10 @@ fromForeignPtrUnsafe fp end bound =
298-- XXX Change the names to use "new" instead of "newArray". That way we can use
366-- XXX Change the names to use "new" instead of "newArray". That way we can use
299-- the same names for managed file system objects as well. For unmanaged ones
367-- the same names for managed file system objects as well. For unmanaged ones
300-- we can use open/create etc as usual.
368-- we can use open/create etc as usual.
370+-- A new array is similar to "touch" creating a zero length file. An mmapped
371+-- array would be similar to a sparse file with holes. TBD: support mmapped
302-- GHC always guarantees word-aligned memory, alignment is important only when
374-- GHC always guarantees word-aligned memory, alignment is important only when
303-- we need more than that. See stg_newAlignedPinnedByteArrayzh and
375-- we need more than that. See stg_newAlignedPinnedByteArrayzh and
@@ -379,6 +451,14 @@ withNewArrayUnsafe count f = do
379-- Random writes
451-- Random writes
380-------------------------------------------------------------------------------
452-------------------------------------------------------------------------------
454+-- | Write an input stream of (index, value) pairs to an array. Throws an
455+-- error if any index is out of bounds.
458+{-# INLINE putIndices #-}
459+putIndices :: Array a -> Fold m (Int, a) ()
460+putIndices = undefined
382-- | Write the given element to the given index of the array. Does not check if
462-- | Write the given element to the given index of the array. Does not check if
383-- the index is out of bounds of the array.
463-- the index is out of bounds of the array.
@@ -411,6 +491,8 @@ putIndexPtr ptr end i x = do
411-- Performs in-place mutation of the array.
491-- Performs in-place mutation of the array.
413-- >>> putIndex arr ix val = Array.modifyIndex arr ix (const (val, ()))
493-- >>> putIndex arr ix val = Array.modifyIndex arr ix (const (val, ()))
494+-- >>> f = Array.putIndices
495+-- >>> putIndex arr ix val = Stream.fold (f arr) (Stream.fromPure (ix, val))
415-- /Pre-release/
497-- /Pre-release/
416{-# INLINE putIndex #-}
498{-# INLINE putIndex #-}
@@ -450,6 +532,27 @@ modifyIndex Array{..} i f = do
450 return res
532 return res
451 else invalidIndex "modifyIndex" i
533 else invalidIndex "modifyIndex" i
535+-- | Modify the array indices generated by the supplied unfold.
538+modifyIndices :: -- forall m a b. (MonadIO m, Storable a) =>
539+ Unfold m (Array a) Int -> Array a -> (a -> a) -> m ()
540+modifyIndices = undefined
542+-- | Modify each element of an array using the supplied modifier function.
545+modify :: -- forall m a b. (MonadIO m, Storable a) =>
546+ Array a -> (a -> a) -> m ()
549+-- | Swap the elements at two indices.
552+swapIndices :: -- (MonadIO m, Storable a) =>
553+ Array a -> Int -> Int -> m ()
554+swapIndices = undefined
453-------------------------------------------------------------------------------
556-------------------------------------------------------------------------------
454-- Rounding
557-- Rounding
455-------------------------------------------------------------------------------
558-------------------------------------------------------------------------------
@@ -497,6 +600,16 @@ arrayChunkSize = mkChunkSize 1024
498-------------------------------------------------------------------------------
601-------------------------------------------------------------------------------
603+-- XXX We can possibly use a smallMutableByteArray to hold the start, end,
604+-- bound pointers. Or we can use the first 16 bytes of the allocation to store
605+-- the end, bound pointers. Using fully mutable handle will ensure that we do
606+-- not have multiple references to the same array of different lengths lying
607+-- around and potentially misused. In that case "snoc" need not return a new
608+-- array (snoc :: Array a -> a -> m ()), it will just modify the old reference.
609+-- The array length will be mutable. This means the length function would also
610+-- be monadic. Mutable arrays would behave more like files that grow in that
500-- | Snoc using a 'Ptr'. Low level reusable function.
613-- | Snoc using a 'Ptr'. Low level reusable function.
502-- /Internal/
615-- /Internal/
@@ -653,7 +766,6 @@ reallocAligned elemSize alignSize newSize Array{..} = do
653 , aBound = pNew `plusPtr` newSize
766 , aBound = pNew `plusPtr` newSize
656−-- XXX can unaligned allocation be more efficient when alignment is not needed?
657{-# INLINABLE realloc #-}
769{-# INLINABLE realloc #-}
658realloc :: forall m a. (MonadIO m, Storable a) => Int -> Array a -> m (Array a)
770realloc :: forall m a. (MonadIO m, Storable a) => Int -> Array a -> m (Array a)
659realloc i arr =
771realloc i arr =
@@ -661,8 +773,30 @@ realloc i arr =
661 $ reallocAligned
773 $ reallocAligned
662 (sizeOf (undefined :: a)) (alignment (undefined :: a)) i arr
774 (sizeOf (undefined :: a)) (alignment (undefined :: a)) i arr
776+-- | Change the reserved memory of the array so that it is enough to hold the
777+-- specified number of elements. Nothing is done if the specified capacity is
778+-- less than the length of the array.
780+-- If the capacity is more than 'largeObjectThreshold' then it is rounded up to
781+-- the block size (4K).
784+{-# INLINE resize #-}
785+resize :: -- (MonadIO m, Storable a) =>
786+ Int -> Array a -> m (Array a)
789+-- | Like 'resize' but if the capacity is more than 'largeObjectThreshold' then
790+-- it is rounded up to the closest power of 2.
793+{-# INLINE resizeExp #-}
794+resizeExp :: -- (MonadIO m, Storable a) =>
795+ Int -> Array a -> m (Array a)
796+resizeExp = undefined
798+-- | Resize the allocated memory to drop any reserved free space at the end of
664−-- | Drop any reserved free space at the end of the array and reallocate it to
799+-- the array and reallocate it to reduce wastage.
667-- Up to 25% wastage is allowed to avoid reallocations. If the capacity is
801-- Up to 25% wastage is allowed to avoid reallocations. If the capacity is
668-- more than 'largeObjectThreshold' then free space up to the 'blockSize' is
802-- more than 'largeObjectThreshold' then free space up to the 'blockSize' is
@@ -686,7 +820,39 @@ rightSize arr@Array{..} = do
686 else return arr
820 else return arr
688-------------------------------------------------------------------------------
822-------------------------------------------------------------------------------
823+-- Reducing the length
824+-------------------------------------------------------------------------------
826+-- | Drop the last n elements of the array to reduce the length by n. The
827+-- capacity is reallocated using the user supplied function.
830+{-# INLINE truncateWith #-}
831+truncateWith :: -- (MonadIO m, Storable a) =>
832+ Int -> (Int -> Int) -> Array a -> m (Array a)
833+truncateWith = undefined
835+-- | Drop the last n elements of the array to reduce the length by n.
837+-- The capacity is rounded to 1K or 4K if the length is more than the GHC large
841+{-# INLINE truncate #-}
842+truncate :: -- (MonadIO m, Storable a) =>
843+ Int -> Array a -> m (Array a)
846+-- | Like 'truncate' but the capacity is rounded to the closest power of 2.
849+{-# INLINE truncateExp #-}
850+truncateExp :: -- (MonadIO m, Storable a) =>
851+ Int -> Array a -> m (Array a)
852+truncateExp = undefined
854+-------------------------------------------------------------------------------
690-------------------------------------------------------------------------------
856-------------------------------------------------------------------------------
692-- | Return the element at the specified index without checking the bounds.
858-- | Return the element at the specified index without checking the bounds.
@@ -772,6 +938,8 @@ getIndices (Unfold stepi injecti) = Unfold step inject
772-- Subarrays
938-- Subarrays
773-------------------------------------------------------------------------------
939-------------------------------------------------------------------------------
941+-- XXX We can also get immutable slices.
775-- | /O(1)/ Slice an array in constant time.
943-- | /O(1)/ Slice an array in constant time.
777-- Unsafe: The bounds of the slice are not checked.
945-- Unsafe: The bounds of the slice are not checked.
@@ -811,6 +979,77 @@ getSlice index len (Array fp e _) =
811 $ "getSlice: invalid slice, index "
979 $ "getSlice: invalid slice, index "
812 ++ show index ++ " length " ++ show len
980 ++ show index ++ " length " ++ show len
982+-------------------------------------------------------------------------------
983+-- In-place mutation algorithms
984+-------------------------------------------------------------------------------
986+-- XXX consider the bulk update/accumulation/permutation APIs from vector.
988+-- | You may not need to reverse an array because you can consume it in reverse
989+-- using 'readRev'. To reverse large arrays you can read in reverse and write
990+-- to another array. However, in-place reverse can be useful to take adavantage
991+-- of cache locality and when you do not want to allocate additional memory.
994+{-# INLINE reverse #-}
995+reverse :: Array a -> m Bool
998+-- | Generate the next permutation of the sequence, returns False if this is
999+-- the last permutation.
1002+{-# INLINE permute #-}
1003+permute :: Array a -> m Bool
1006+-- | Partition an array into two halves using a partitioning predicate. The
1007+-- first half retains values where the predicate is 'False' and the second half
1008+-- retains values where the predicate is 'True'.
1011+{-# INLINE partitionBy #-}
1012+partitionBy :: (a -> Bool) -> Array a -> m (Array a, Array a)
1013+partitionBy = undefined
1015+-- | Shuffle corresponding elements from two arrays using a shuffle function.
1016+-- If the shuffle function returns 'False' then do nothing otherwise swap the
1017+-- elements. This can be used in a bottom up fold to shuffle or reorder the
1021+{-# INLINE shuffleBy #-}
1022+shuffleBy :: (a -> a -> m Bool) -> Array a -> Array a -> m (Array a)
1023+shuffleBy = undefined
1025+-- XXX we can also make the folds partial by stopping at a certain level.
1027+-- | @divideBy level partition array@ performs a top down hierarchical
1028+-- recursive partitioning fold of items in the container using the given
1029+-- function as the partition function. Level indicates the level in the tree
1030+-- where the fold would stop.
1032+-- This performs a quick sort if the partition function is
1033+-- 'partitionBy (< pivot)'.
1036+{-# INLINABLE divideBy #-}
1038+ Int -> (Array a -> Array a -> m (Array a)) -> Array a -> m (Array a)
1039+divideBy = undefined
1041+-- | @mergeBy level merge array@ performs a pairwise bottom up fold recursively
1042+-- merging the pairs using the supplied merge function. Level indicates the
1043+-- level in the tree where the fold would stop.
1045+-- This performs a random shuffle if the shuffle function is random. If we
1046+-- stop at level 0 and repeatedly apply the function then we can do a bubble
1050+mergeBy :: Int -> (Array a -> Array a -> m (Array a)) -> Array a -> m (Array a)
814-------------------------------------------------------------------------------
1053-------------------------------------------------------------------------------
816-------------------------------------------------------------------------------
1055-------------------------------------------------------------------------------