1{-# LANGUAGE UnboxedTuples #-}
6-- Module : Streamly.Internal.Memory.Array
7-- Copyright : (c) 2019 Composewell Technologies
10-- Maintainer : streamly@composewell.com
11-- Stability : experimental
16-- * Arrays are finite and fixed in size
17-- * provide /O(1)/ access to elements
18-- * store only data and not functions
19-- * provide efficient IO interfacing
21-- 'Foldable' instance is not provided because the implementation would be much
22-- less efficient compared to folding via streams. 'Semigroup' and 'Monoid'
23-- instances should be used with care; concatenating arrays using binary
24-- operations can be highly inefficient. Instead, use
25-- 'Streamly.Internal.Memory.ArrayStream.toArray' to concatenate N arrays at
28-- Each array is one pointer visible to the GC. Too many small arrays (e.g.
29-- single byte) are only as good as holding those elements in a Haskell list.
30-- However, small arrays can be compacted into large ones to reduce the
31-- overhead. To hold 32GB memory in 32k sized buffers we need 1 million arrays
32-- if we use one array for each chunk. This is still significant to add
35module Streamly.Internal.Memory.Array
53 , A.writeN -- drop new
55 , A.write -- full buffer
56 -- , writeLastN -- drop old (ring buffer)
78 -- , readFrom -- read from a given position to the end of file
79 -- , readFromRev -- read from a given position to the beginning of file
80 -- , readTo -- read from beginning up to the given position
81 -- , readToRev -- read from end to the given position in file
89 -- , writeFrom -- start writing at the given position
91 -- , writeTo -- write from beginning up to the given position
96 -- , writeChunksOfFrom
110 -- -- * In-pace mutation (for Mutable Array type)
116 -- * Immutable Transformations
123 -- * Folds with Array as the container
128import Control.Monad (when)
129import Control.Monad.IO.Class (MonadIO(..))
130-- import Data.Functor.Identity (Identity)
131import Foreign.ForeignPtr (withForeignPtr, touchForeignPtr)
132import Foreign.Ptr (plusPtr)
133import Foreign.Storable (Storable(..))
134import Prelude hiding (length, null, last, map, (!!), read, concat)
136import GHC.ForeignPtr (ForeignPtr(..))
137import GHC.Ptr (Ptr(..))
138import GHC.Prim (touch#)
139import GHC.IO (IO(..))
141import Streamly.Internal.Data.Fold.Types (Fold(..))
142import Streamly.Internal.Data.Unfold.Types (Unfold(..))
143import Streamly.Internal.Memory.Array.Types (Array(..), length)
144import Streamly.Internal.Data.Stream.Serial (SerialT)
145import Streamly.Internal.Data.Stream.StreamK.Type (IsStream)
147import qualified Streamly.Internal.Memory.Array.Types as A
148import qualified Streamly.Internal.Data.Stream.Prelude as P
149import qualified Streamly.Internal.Data.Stream.Serial as Serial
150import qualified Streamly.Internal.Data.Stream.StreamD as D
151import qualified Streamly.Internal.Data.Stream.StreamK as K
153-------------------------------------------------------------------------------
155-------------------------------------------------------------------------------
158-- | Create a new uninitialized array of given length.
161newArray :: (MonadIO m, Storable a) => Int -> m (Array a)
162newArray len = undefined
165-- | Create an 'Array' from the first N elements of a stream. The array is
166-- allocated to size N, if the stream terminates before N elements then the
167-- array may hold less than N elements.
170{-# INLINE fromStreamN #-}
171fromStreamN :: (MonadIO m, Storable a) => Int -> SerialT m a -> m (Array a)
173 when (n < 0) $ error "writeN: negative write count specified"
174 A.fromStreamDN n $ D.toStreamD m
176-- | Create an 'Array' from a stream. This is useful when we want to create a
177-- single array from a stream of unknown size. 'writeN' is at least twice
178-- as efficient when the size is already known.
180-- Note that if the input stream is too large memory allocation for the array
181-- may fail. When the stream size is not known, `arraysOf` followed by
182-- processing of indvidual arrays in the resulting stream should be preferred.
185{-# INLINE fromStream #-}
186fromStream :: (MonadIO m, Storable a) => SerialT m a -> m (Array a)
187fromStream = P.runFold A.write
188-- write m = A.fromStreamD $ D.toStreamD m
190-------------------------------------------------------------------------------
192-------------------------------------------------------------------------------
194-- | Convert an 'Array' into a stream.
197{-# INLINE_EARLY toStream #-}
198toStream :: (Monad m, K.IsStream t, Storable a) => Array a -> t m a
199toStream = D.fromStreamD . A.toStreamD
200-- XXX add fallback to StreamK rule
201-- {-# RULES "Streamly.Array.read fallback to StreamK" [1]
202-- forall a. S.readK (read a) = K.fromArray a #-}
204-- | Convert an 'Array' into a stream in reverse order.
207{-# INLINE_EARLY toStreamRev #-}
208toStreamRev :: (Monad m, IsStream t, Storable a) => Array a -> t m a
209toStreamRev = D.fromStreamD . A.toStreamDRev
210-- XXX add fallback to StreamK rule
211-- {-# RULES "Streamly.Array.readRev fallback to StreamK" [1]
212-- forall a. S.toStreamK (readRev a) = K.revFromArray a #-}
214data ReadUState a = ReadUState
215 {-# UNPACK #-} !(ForeignPtr a) -- foreign ptr with end of array pointer
216 {-# UNPACK #-} !(Ptr a) -- current pointer
218-- | Unfold an array into a stream.
221{-# INLINE_NORMAL read #-}
222read :: forall m a. (Monad m, Storable a) => Unfold m (Array a) a
223read = Unfold step inject
226 inject (Array (ForeignPtr start contents) (Ptr end) _) =
227 return $ ReadUState (ForeignPtr end contents) (Ptr start)
229 {-# INLINE_LATE step #-}
230 step (ReadUState fp@(ForeignPtr end _) p) | p == Ptr end =
231 let x = A.unsafeInlineIO $ touchForeignPtr fp
232 in x `seq` return D.Stop
233 step (ReadUState fp p) = do
234 -- unsafeInlineIO allows us to run this in Identity monad for pure
235 -- toList/foldr case which makes them much faster due to not
236 -- accumulating the list and fusing better with the pure consumers.
238 -- This should be safe as the array contents are guaranteed to be
239 -- evaluated/written to before we peek at them.
240 let !x = A.unsafeInlineIO $ peek p
242 (ReadUState fp (p `plusPtr` sizeOf (undefined :: a)))
244-- | Unfold an array into a stream, does not check the end of the array, the
245-- user is responsible for terminating the stream within the array bounds. For
246-- high performance application where the end condition can be determined by
247-- a terminating fold.
249-- Written in the hope that it may be faster than "read", however, in the case
250-- for which this was written, "read" proves to be faster even though the core
251-- generated with unsafeRead looks simpler.
255{-# INLINE_NORMAL unsafeRead #-}
256unsafeRead :: forall m a. (Monad m, Storable a) => Unfold m (Array a) a
257unsafeRead = Unfold step inject
260 inject (Array fp _ _) = return fp
262 {-# INLINE_LATE step #-}
263 step (ForeignPtr p contents) = do
264 -- unsafeInlineIO allows us to run this in Identity monad for pure
265 -- toList/foldr case which makes them much faster due to not
266 -- accumulating the list and fusing better with the pure consumers.
268 -- This should be safe as the array contents are guaranteed to be
269 -- evaluated/written to before we peek at them.
270 let !x = A.unsafeInlineIO $ do
274 let !(Ptr p1) = Ptr p `plusPtr` sizeOf (undefined :: a)
275 return $ D.Yield x (ForeignPtr p1 contents)
277 touch r = IO $ \s -> case touch# r s of s' -> (# s', () #)
279-- | > null arr = length arr == 0
283null :: Storable a => Array a -> Bool
284null arr = length arr == 0
286-- | > last arr = readIndex arr (length arr - 1)
290last :: Storable a => Array a -> Maybe a
291last arr = readIndex arr (length arr - 1)
293-------------------------------------------------------------------------------
295-------------------------------------------------------------------------------
297-------------------------------------------------------------------------------
299-------------------------------------------------------------------------------
302-- | Perform a binary search in the array to find an element.
303bsearch :: a -> Array a -> Maybe Bool
306-- | Perform a binary search in the array to find an element index.
307{-# INLINE elemIndex #-}
308bsearchIndex :: a -> Array a -> Maybe Int
309bsearchIndex elem arr = undefined
311-- find/findIndex etc can potentially be implemented more efficiently on arrays
312-- compared to streams by using SIMD instructions.
314find :: (a -> Bool) -> Array a -> Bool
317findIndex :: (a -> Bool) -> Array a -> Maybe Int
320findIndices :: (a -> Bool) -> Array a -> Array Int
321findIndices = undefined
324-------------------------------------------------------------------------------
326-------------------------------------------------------------------------------
328-- XXX We can potentially use SIMD instructions on arrays to fold faster.
330-------------------------------------------------------------------------------
332-------------------------------------------------------------------------------
335slice :: Int -> Int -> Array a
336slice begin end arr = undefined
338splitAt :: Int -> Array a -> (Array a, Array a)
339splitAt i arr = undefined
341-- XXX This operation can be performed efficiently via streams.
342-- | Append two arrays together to create a single array.
343splice :: Array a -> Array a -> Array a
344splice arr1 arr2 = undefined
346-------------------------------------------------------------------------------
347-- In-place mutation APIs
348-------------------------------------------------------------------------------
350-- | Partition an array into two halves using a partitioning predicate. The
351-- first half retains values where the predicate is 'False' and the second half
352-- retains values where the predicate is 'True'.
353{-# INLINE partitionBy #-}
354partitionBy :: (a -> Bool) -> Array a -> (Array a, Array a)
355partitionBy f arr = undefined
357-- | Shuffle corresponding elements from two arrays using a shuffle function.
358-- If the shuffle function returns 'False' then do nothing otherwise swap the
359-- elements. This can be used in a bottom up fold to shuffle or reorder the
361shuffleBy :: (a -> a -> m Bool) -> Array a -> Array a -> m (Array a)
362shuffleBy f arr1 arr2 = undefined
364-- XXX we can also make the folds partial by stopping at a certain level.
366-- | Perform a top down hierarchical recursive partitioning fold of items in
367-- the container using the given function as the partition function.
369-- This will perform a quick sort if the partition function is
370-- 'partitionBy (< pivot)'.
373{-# INLINABLE foldtWith #-}
374foldtWith :: Int -> (Array a -> Array a -> m (Array a)) -> Array a -> m (Array a)
375foldtWith level f = undefined
377-- | Perform a pairwise bottom up fold recursively merging the pairs. Level
378-- indicates the level in the tree where the fold would stop.
380-- This will perform a random shuffle if the shuffle function is random.
381-- If we stop at level 0 and repeatedly apply the function then we can do a
383foldbWith :: Int -> (Array a -> Array a -> m (Array a)) -> Array a -> m (Array a)
384foldbWith level f = undefined
387-- XXX consider the bulk update/accumulation/permutation APIs from vector.
389-------------------------------------------------------------------------------
390-- Random reads and writes
391-------------------------------------------------------------------------------
393-- | /O(1)/ Lookup the element at the given index, starting from 0.
396{-# INLINE readIndex #-}
397readIndex :: Storable a => Array a -> Int -> Maybe a
399 if i < 0 || i > length arr - 1
401 else A.unsafeInlineIO $
402 withForeignPtr (aStart arr) $ \p -> Just <$> peekElemOff p i
405-- | @readSlice arr i count@ streams a slice of the array @arr@ starting
406-- at index @i@ and reading up to @count@ elements in the forward direction
407-- ending at the index @i + count - 1@.
410{-# INLINE readSlice #-}
411readSlice :: (IsStream t, Monad m, Storable a)
412 => Array a -> Int -> Int -> t m a
413readSlice arr i len = undefined
415-- | @readSliceRev arr i count@ streams a slice of the array @arr@ starting at
416-- index @i@ and reading up to @count@ elements in the reverse direction ending
417-- at the index @i - count + 1@.
420{-# INLINE readSliceRev #-}
421readSliceRev :: (IsStream t, Monad m, Storable a)
422 => Array a -> Int -> Int -> t m a
423readSliceRev arr i len = undefined
426-- | /O(1)/ Write the given element at the given index in the array.
427-- Performs in-place mutation of the array.
430{-# INLINE writeIndex #-}
431writeIndex :: (MonadIO m, Storable a) => Array a -> Int -> a -> m ()
432writeIndex arr i a = do
433 let maxIndex = length arr - 1
435 then error "writeIndex: negative array index"
437 then error $ "writeIndex: specified array index " ++ show i
438 ++ " is beyond the maximum index " ++ show maxIndex
440 liftIO $ withForeignPtr (aStart arr) $ \p ->
444-- | @writeSlice arr i count stream@ writes a stream to the array @arr@
445-- starting at index @i@ and writing up to @count@ elements in the forward
446-- direction ending at the index @i + count - 1@.
449{-# INLINE writeSlice #-}
450writeSlice :: (IsStream t, Monad m, Storable a)
451 => Array a -> Int -> Int -> t m a -> m ()
452writeSlice arr i len s = undefined
454-- | @writeSliceRev arr i count stream@ writes a stream to the array @arr@
455-- starting at index @i@ and writing up to @count@ elements in the reverse
456-- direction ending at the index @i - count + 1@.
459{-# INLINE writeSliceRev #-}
460writeSliceRev :: (IsStream t, Monad m, Storable a)
461 => Array a -> Int -> Int -> t m a -> m ()
462writeSliceRev arr i len s = undefined
465-------------------------------------------------------------------------------
466-- Transform via stream operations
467-------------------------------------------------------------------------------
469-- for non-length changing operations we can use the original length for
470-- allocation. If we can predict the length then we can use the prediction for
471-- new allocation. Otherwise we can use a hint and adjust dynamically.
474-- | Transform an array into another array using a pipe transformation
478{-# INLINE runPipe #-}
479runPipe :: (MonadIO m, Storable a, Storable b)
480 => Pipe m a b -> Array a -> m (Array b)
481runPipe f arr = P.runPipe (toArrayMinChunk (length arr)) $ f (A.read arr)
484-- | Transform an array into another array using a stream transformation
488{-# INLINE streamTransform #-}
489streamTransform :: forall m a b. (MonadIO m, Storable a, Storable b)
490 => (SerialT m a -> SerialT m b) -> Array a -> m (Array b)
491streamTransform f arr =
492 P.runFold (A.toArrayMinChunk (alignment (undefined :: a)) (length arr))
495-- | Fold an array using a 'Fold'.
499fold :: forall m a b. (MonadIO m, Storable a) => Fold m a b -> Array a -> m b
500fold f arr = P.runFold f (toStream arr :: Serial.SerialT m a)
502-- | Fold an array using a stream fold operation.
505{-# INLINE streamFold #-}
506streamFold :: (MonadIO m, Storable a) => (SerialT m a -> m b) -> Array a -> m b
507streamFold f arr = f (toStream arr)
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list