@@ -0,0 +1,392 @@
1+{-# LANGUAGE BangPatterns #-}
2+{-# LANGUAGE CPP #-}
3+{-# LANGUAGE MagicHash #-}
4+{-# LANGUAGE RecordWildCards #-}
5+{-# LANGUAGE UnboxedTuples #-}
6+{-# LANGUAGE ScopedTypeVariables #-}
7+
8+#include "inline.hs"
9+
10+-- |
11+-- Module : Streamly.Internal.Memory.Array
12+-- Copyright : (c) 2019 Composewell Technologies
13+--
14+-- License : BSD3
15+-- Maintainer : streamly@composewell.com
16+-- Stability : experimental
17+-- Portability : GHC
18+--
19+-- To summarize:
20+--
21+-- * Arrays are finite and fixed in size
22+-- * provide /O(1)/ access to elements
23+-- * store only data and not functions
24+-- * provide efficient IO interfacing
25+--
26+-- 'Foldable' instance is not provided because the implementation would be much
27+-- less efficient compared to folding via streams. 'Semigroup' and 'Monoid'
28+-- instances should be used with care; concatenating arrays using binary
29+-- operations can be highly inefficient. Instead, use
30+-- 'Streamly.Internal.Memory.ArrayStream.toArray' to concatenate N arrays at
31+-- once.
32+--
33+-- Each array is one pointer visible to the GC. Too many small arrays (e.g.
34+-- single byte) are only as good as holding those elements in a Haskell list.
35+-- However, small arrays can be compacted into large ones to reduce the
36+-- overhead. To hold 32GB memory in 32k sized buffers we need 1 million arrays
37+-- if we use one array for each chunk. This is still significant to add
38+-- pressure to GC.
39+
40+module Streamly.Internal.Memory.Array
41+ (
42+ Array
43+
44+ -- , defaultChunkSize
45+
46+ -- * Arrays
47+ -- ** Construction
48+ -- | When performance matters, the fastest way to generate an array is
49+ -- 'writeN'. For regular use, 'IsList' and 'IsString' instances can be
50+ -- used to conveniently construct arrays from literal values.
51+ -- 'OverloadedLists' extension or 'fromList' can be used to construct an
52+ -- array from a list literal. Similarly, 'OverloadedStrings' extension or
53+ -- 'fromList' can be used to construct an array from a string literal.
54+
55+ -- Pure List APIs
56+ , A.fromListN
57+ , A.fromList
58+
59+ -- Monadic APIs
60+ -- , newArray
61+ , A.writeN -- drop new
62+ , A.write -- full buffer
63+ -- , writeLastN -- drop old (ring buffer)
64+
65+ -- Stream Folds
66+ -- , writeNS
67+ -- , writeS
68+
69+ -- ** Elimination
70+ -- 'GHC.Exts.toList' from "GHC.Exts" can be used to convert an array to a
71+ -- list.
72+
73+ , A.toList
74+ , A.read
75+ , readRev
76+
77+ -- ** Random Access
78+ , length
79+ -- , (!!)
80+
81+ , readIndex
82+ {-
83+ , readSlice
84+ , readSliceRev
85+ -}
86+
87+ -- , writeIndex
88+ {-
89+ , writeSlice
90+ , writeSliceRev
91+ -}
92+
93+ {-
94+ -- * Immutable Transformations
95+ , runTransform
96+
97+ -- * Folding Arrays
98+ -- , runStreamFold
99+ , runFold
100+ -}
101+ )
102+where
103+
104+import Control.Monad.IO.Class (MonadIO(..))
105+-- import Data.Functor.Identity (Identity)
106+import Foreign.ForeignPtr (withForeignPtr)
107+import Foreign.Storable (Storable(..))
108+import Prelude hiding (length, null, last, map, (!!), read, concat)
109+
110+import Streamly.Internal.Memory.Array.Types (Array(..), length)
111+import Streamly.Streams.Serial (SerialT)
112+import Streamly.Streams.StreamK.Type (IsStream)
113+import Streamly.Internal.Data.Fold.Types (Fold(..))
114+
115+import qualified Streamly.Internal.Memory.Array.Types as A
116+import qualified Streamly.Streams.Prelude as P
117+import qualified Streamly.Streams.Serial as Serial
118+import qualified Streamly.Streams.StreamD as D
119+
120+-------------------------------------------------------------------------------
121+-- Construction
122+-------------------------------------------------------------------------------
123+
124+{-
125+-- | Create a new uninitialized array of given length.
126+--
127+-- @since 0.7.0
128+newArray :: (MonadIO m, Storable a) => Int -> m (Array a)
129+newArray len = undefined
130+-}
131+
132+-- | Create an 'Array' from the first N elements of a stream. The array is
133+-- allocated to size N, if the stream terminates before N elements then the
134+-- array may hold less than N elements.
135+--
136+-- @since 0.7.0
137+{-# INLINE _writeNS #-}
138+_writeNS :: (MonadIO m, Storable a) => Int -> SerialT m a -> m (Array a)
139+_writeNS n m = do
140+ if n < 0 then error "writeN: negative write count specified" else return ()
141+ A.fromStreamDN n $ D.toStreamD m
142+
143+-- | Create an 'Array' from a stream. This is useful when we want to create a
144+-- single array from a stream of unknown size. 'writeN' is at least twice
145+-- as efficient when the size is already known.
146+--
147+-- Note that if the input stream is too large memory allocation for the array
148+-- may fail. When the stream size is not known, `arraysOf` followed by
149+-- processing of indvidual arrays in the resulting stream should be preferred.
150+--
151+{-# INLINE _writeS #-}
152+_writeS :: (MonadIO m, Storable a) => SerialT m a -> m (Array a)
153+_writeS = P.runFold A.write
154+-- write m = A.fromStreamD $ D.toStreamD m
155+
156+-------------------------------------------------------------------------------
157+-- Elimination
158+-------------------------------------------------------------------------------
159+
160+-- | Convert an 'Array' into a stream in reverse order.
161+--
162+-- @since 0.7.0
163+{-# INLINE_EARLY readRev #-}
164+readRev :: (Monad m, IsStream t, Storable a) => Array a -> t m a
165+readRev = D.fromStreamD . A.toStreamDRev
166+-- XXX add fallback to StreamK rule
167+-- {-# RULES "Streamly.Array.readRev fallback to StreamK" [1]
168+-- forall a. S.toStreamK (readRev a) = K.revFromArray a #-}
169+
170+{-# INLINE _null #-}
171+_null :: Storable a => Array a -> Bool
172+_null arr = length arr == 0
173+
174+{-# INLINE _last #-}
175+_last :: Storable a => Array a -> Maybe a
176+_last arr = readIndex arr (length arr - 1)
177+
178+-------------------------------------------------------------------------------
179+-- Random Access
180+-------------------------------------------------------------------------------
181+
182+-------------------------------------------------------------------------------
183+-- Searching
184+-------------------------------------------------------------------------------
185+
186+{-
187+-- | Perform a binary search in the array to find an element.
188+bsearch :: a -> Array a -> Maybe Bool
189+bsearch = undefined
190+
191+-- | Perform a binary search in the array to find an element index.
192+{-# INLINE elemIndex #-}
193+bsearchIndex :: a -> Array a -> Maybe Int
194+bsearchIndex elem arr = undefined
195+
196+-- find/findIndex etc can potentially be implemented more efficiently on arrays
197+-- compared to streams by using SIMD instructions.
198+
199+find :: (a -> Bool) -> Array a -> Bool
200+find = undefined
201+
202+findIndex :: (a -> Bool) -> Array a -> Maybe Int
203+findIndex = undefined
204+
205+findIndices :: (a -> Bool) -> Array a -> Array Int
206+findIndices = undefined
207+-}
208+
209+-------------------------------------------------------------------------------
210+-- Folds
211+-------------------------------------------------------------------------------
212+
213+-- XXX We can potentially use SIMD instructions on arrays to fold faster.
214+
215+-------------------------------------------------------------------------------
216+-- Slice and splice
217+-------------------------------------------------------------------------------
218+
219+{-
220+slice :: Int -> Int -> Array a
221+slice begin end arr = undefined
222+
223+splitAt :: Int -> Array a -> (Array a, Array a)
224+splitAt i arr = undefined
225+
226+-- XXX This operation can be performed efficiently via streams.
227+-- | Append two arrays together to create a single array.
228+splice :: Array a -> Array a -> Array a
229+splice arr1 arr2 = undefined
230+
231+-------------------------------------------------------------------------------
232+-- In-place mutation APIs
233+-------------------------------------------------------------------------------
234+
235+-- | Partition an array into two halves using a partitioning predicate. The
236+-- first half retains values where the predicate is 'False' and the second half
237+-- retains values where the predicate is 'True'.
238+{-# INLINE partitionBy #-}
239+partitionBy :: (a -> Bool) -> Array a -> (Array a, Array a)
240+partitionBy f arr = undefined
241+
242+-- | Shuffle corresponding elements from two arrays using a shuffle function.
243+-- If the shuffle function returns 'False' then do nothing otherwise swap the
244+-- elements. This can be used in a bottom up fold to shuffle or reorder the
245+-- elements.
246+shuffleBy :: (a -> a -> m Bool) -> Array a -> Array a -> m (Array a)
247+shuffleBy f arr1 arr2 = undefined
248+
249+-- XXX we can also make the folds partial by stopping at a certain level.
250+--
251+-- | Perform a top down hierarchical recursive partitioning fold of items in
252+-- the container using the given function as the partition function.
253+--
254+-- This will perform a quick sort if the partition function is
255+-- 'partitionBy (< pivot)'.
256+--
257+-- @since 0.7.0
258+{-# INLINABLE foldtWith #-}
259+foldtWith :: Int -> (Array a -> Array a -> m (Array a)) -> Array a -> m (Array a)
260+foldtWith level f = undefined
261+
262+-- | Perform a pairwise bottom up fold recursively merging the pairs. Level
263+-- indicates the level in the tree where the fold would stop.
264+--
265+-- This will perform a random shuffle if the shuffle function is random.
266+-- If we stop at level 0 and repeatedly apply the function then we can do a
267+-- bubble sort.
268+foldbWith :: Int -> (Array a -> Array a -> m (Array a)) -> Array a -> m (Array a)
269+foldbWith level f = undefined
270+-}
271+
272+-- XXX consider the bulk update/accumulation/permutation APIs from vector.
273+
274+-------------------------------------------------------------------------------
275+-- Random reads and writes
276+-------------------------------------------------------------------------------
277+
278+-- | /O(1)/ Lookup the element at the given index, starting from 0.
279+--
280+-- @since 0.7.0
281+{-# INLINE readIndex #-}
282+readIndex :: Storable a => Array a -> Int -> Maybe a
283+readIndex arr i =
284+ if i < 0 || i > length arr - 1
285+ then Nothing
286+ else A.unsafeInlineIO $
287+ withForeignPtr (aStart arr) $ \p -> fmap Just $ peekElemOff p i
288+
289+{-
290+-- | @readSlice arr i count@ streams a slice of the array @arr@ starting
291+-- at index @i@ and reading up to @count@ elements in the forward direction
292+-- ending at the index @i + count - 1@.
293+--
294+-- @since 0.7.0
295+{-# INLINE readSlice #-}
296+readSlice :: (IsStream t, Monad m, Storable a)
297+ => Array a -> Int -> Int -> t m a
298+readSlice arr i len = undefined
299+
300+-- | @readSliceRev arr i count@ streams a slice of the array @arr@ starting at
301+-- index @i@ and reading up to @count@ elements in the reverse direction ending
302+-- at the index @i - count + 1@.
303+--
304+-- @since 0.7.0
305+{-# INLINE readSliceRev #-}
306+readSliceRev :: (IsStream t, Monad m, Storable a)
307+ => Array a -> Int -> Int -> t m a
308+readSliceRev arr i len = undefined
309+-}
310+
311+-- | /O(1)/ Write the given element at the given index in the array.
312+-- Performs in-place mutation of the array.
313+--
314+-- @since 0.7.0
315+{-# INLINE _writeIndex #-}
316+_writeIndex :: (MonadIO m, Storable a) => Array a -> Int -> a -> m ()
317+_writeIndex arr i a = do
318+ let maxIndex = length arr - 1
319+ if i < 0
320+ then error "writeIndex: negative array index"
321+ else if i > maxIndex
322+ then error $ "writeIndex: specified array index " ++ show i
323+ ++ " is beyond the maximum index " ++ show maxIndex
324+ else
325+ liftIO $ withForeignPtr (aStart arr) $ \p ->
326+ pokeElemOff p i a
327+
328+{-
329+-- | @writeSlice arr i count stream@ writes a stream to the array @arr@
330+-- starting at index @i@ and writing up to @count@ elements in the forward
331+-- direction ending at the index @i + count - 1@.
332+--
333+-- @since 0.7.0
334+{-# INLINE writeSlice #-}
335+writeSlice :: (IsStream t, Monad m, Storable a)
336+ => Array a -> Int -> Int -> t m a -> m ()
337+writeSlice arr i len s = undefined
338+
339+-- | @writeSliceRev arr i count stream@ writes a stream to the array @arr@
340+-- starting at index @i@ and writing up to @count@ elements in the reverse
341+-- direction ending at the index @i - count + 1@.
342+--
343+-- @since 0.7.0
344+{-# INLINE writeSliceRev #-}
345+writeSliceRev :: (IsStream t, Monad m, Storable a)
346+ => Array a -> Int -> Int -> t m a -> m ()
347+writeSliceRev arr i len s = undefined
348+-}
349+
350+-------------------------------------------------------------------------------
351+-- Transform via stream operations
352+-------------------------------------------------------------------------------
353+
354+-- for non-length changing operations we can use the original length for
355+-- allocation. If we can predict the length then we can use the prediction for
356+-- new allocation. Otherwise we can use a hint and adjust dynamically.
357+
358+{-
359+-- | Transform an array into another array using a pipe transformation
360+-- operation.
361+--
362+-- @since 0.7.0
363+{-# INLINE runPipe #-}
364+runPipe :: (MonadIO m, Storable a, Storable b)
365+ => Pipe m a b -> Array a -> m (Array b)
366+runPipe f arr = P.runPipe (toArrayMinChunk (length arr)) $ f (A.read arr)
367+-}
368+
369+-- | Transform an array into another array using a stream transformation
370+-- operation.
371+--
372+-- @since 0.7.0
373+{-# INLINE _runTransform #-}
374+_runTransform :: forall m a b. (MonadIO m, Storable a, Storable b)
375+ => (SerialT m a -> SerialT m b) -> Array a -> m (Array b)
376+_runTransform f arr =
377+ P.runFold (A.toArrayMinChunk (alignment (undefined :: a)) (length arr))
378+ $ f (A.read arr)
379+
380+-- | Fold an array using a 'Fold'.
381+--
382+-- @since 0.7.0
383+{-# INLINE _runFold #-}
384+_runFold :: forall m a b. (MonadIO m, Storable a) => Fold m a b -> Array a -> m b
385+_runFold f arr = P.runFold f $ (A.read arr :: Serial.SerialT m a)
386+
387+-- | Fold an array using a stream fold operation.
388+--
389+-- @since 0.7.0
390+{-# INLINE _runStreamFold #-}
391+_runStreamFold :: (MonadIO m, Storable a) => (SerialT m a -> m b) -> Array a -> m b
392+_runStreamFold f arr = f (A.read arr)