@@ -31,7 +31,6 @@ import Streamly.Internal.Data.Unbox
3131 ( MutableByteArray(..)
3232 , PinnedState(..)
3333 , Unbox
34− , sizeOfMutableByteArray
3534 )
3635import Streamly.Internal.Data.Array.Type (Array(..))
3736import Streamly.Internal.System.IO (unsafeInlineIO)
@@ -83,9 +82,9 @@ newtype Size a = Size (Int -> a -> Int) -- a left fold or Sum monoid
8382-- (Size f, Size g) ->
8483-- Size $ \acc obj ->
8584-- acc + f 0 (_obj1 obj) + g 0 (_obj2 obj)
86−-- deserialize i arr = do
87−-- (i1, x0) <- deserialize i arr
88−-- (i2, x1) <- deserialize i1 arr
85+-- deserialize i arr len = do
86+-- (i1, x0) <- deserialize i arr len
87+-- (i2, x1) <- deserialize i1 arr len
8988-- pure (i2, Object x0 x1)
9089-- serialize i arr (Object x0 x1) = do
9190-- i1 <- serialize i arr x0
@@ -102,12 +101,16 @@ class Serialize a where
102101 -- offset but that may require traversing the Haskell structure again to get
103102 -- the size. Therefore, this is a performance optimization.
104103
105− -- | Deserialize a value from the given byte-index in the array. Returns a
106− -- tuple of the next byte-index and the deserialized value.
107− deserialize :: Int -> MutableByteArray -> IO (Int, a)
104+ -- | @deserialize offset array arrayLen@ deserializes a value from the
105+ -- given byte-index in the array. Returns a tuple of the next byte-index
106+ -- and the deserialized value.
107+ deserialize :: Int -> MutableByteArray -> Int -> IO (Int, a)
108108
109109 -- | Write the serialized representation of the value in the array at the
110− -- given byte-index. Returns the next byte-index.
110+ -- given byte-index. Returns the next byte-index. This is an unsafe
111+ -- operation, the programmer must ensure that the array has enough space
112+ -- available in the array to serialize the value as determined by the
113+ -- @size@ operation.
111114 serialize :: Int -> MutableByteArray -> a -> IO Int
112115
113116--------------------------------------------------------------------------------
@@ -132,12 +135,29 @@ checkBounds _label _off _arr = do
132135 else return ()
133136#endif
134137
138+-- Note: Instead of passing around the size parameter, we can use
139+-- (sizeOfMutableByteArray arr) for checking the array bound, but that turns
140+-- out to be more expensive.
141+--
142+-- Another way to optimize this is to avoid the check for fixed size
143+-- structures. For fixed size structures we can do a check at the top level and
144+-- then use checkless deserialization using the Unbox type class. That will
145+-- require ConstSize and VarSize constructors in size. The programmer can
146+-- bundle all const size fields in a newtype to make serialization faster. This
147+-- can speed up the computation of size when serializing and checking size when
148+-- deserialing.
149+--
150+-- For variable size non-recursive structures a separate size validation method
151+-- could be used to validate the size before deserializing. "validate" can also
152+-- be used to collpase multiple chunks of arrays coming from network into a
153+-- single array for deserializing. But that can also be done by framing the
154+-- serialized value with a size header.
155+--
135156{-# INLINE deserializeUnsafe #-}
136−deserializeUnsafe :: forall a. Unbox a => Int -> MutableByteArray -> IO (Int, a)
137−deserializeUnsafe off arr =
157+deserializeUnsafe :: forall a. Unbox a => Int -> MutableByteArray -> Int -> IO (Int, a)
158+deserializeUnsafe off arr sz =
138159 let next = off + Unbox.sizeOf (Proxy :: Proxy a)
139160 in do
140− sz <- sizeOfMutableByteArray arr
141161 -- Keep likely path in the straight branch.
142162 if (next <= sz)
143163 then Unbox.peekByteIndex off arr >>= \val -> pure (next, val)
@@ -162,7 +182,7 @@ instance Serialize _type where \
162182; {-# INLINE size #-} \
163183; size = Size (\acc _ -> acc + Unbox.sizeOf (Proxy :: Proxy _type)) \
164184; {-# INLINE deserialize #-} \
165−; deserialize off arr = deserializeUnsafe off arr :: IO (Int, _type) \
185+; deserialize off arr end = deserializeUnsafe off arr end :: IO (Int, _type) \
166186; {-# INLINE serialize #-} \
167187; serialize = \
168188 serializeUnsafe :: Int -> MutableByteArray -> _type -> IO Int
@@ -193,19 +213,19 @@ instance forall a. Serialize a => Serialize [a] where
193213 Size f -> foldl' f (acc + (Unbox.sizeOf (Proxy :: Proxy Int))) xs
194214
195215 {-# INLINE deserialize #-}
196− deserialize off arr = do
216+ deserialize off arr sz = do
197217 len <- Unbox.peekByteIndex off arr :: IO Int
198218 let off1 = off + Unbox.sizeOf (Proxy :: Proxy Int)
199219 let
200220 peekList f o i | i >= 3 = do
201221 -- Unfold the loop three times
202− (o1, x1) <- deserialize o arr
203− (o2, x2) <- deserialize o1 arr
204− (o3, x3) <- deserialize o2 arr
222+ (o1, x1) <- deserialize o arr sz
223+ (o2, x2) <- deserialize o1 arr sz
224+ (o3, x3) <- deserialize o2 arr sz
205225 peekList (f . (\xs -> x1:x2:x3:xs)) o3 (i - 3)
206226 peekList f o 0 = pure (o, f [])
207227 peekList f o i = do
208− (o1, x) <- deserialize o arr
228+ (o1, x) <- deserialize o arr sz
209229 peekList (f . (x:)) o1 (i - 1)
210230 peekList id off1 len
211231
@@ -250,8 +270,8 @@ pinnedEncode = encodeAs Pinned
250270decode :: Serialize a => Array Word8 -> a
251271decode arr@(Array {..}) = unsafeInlineIO $ do
252272 let lenArr = Array.length arr
253− (off1, lenEncoding :: Int64) <- deserialize 0 arrContents
254− (off2, val) <- deserialize off1 arrContents
273+ (off1, lenEncoding :: Int64) <- deserialize 0 arrContents lenArr
274+ (off2, val) <- deserialize off1 arrContents lenArr
255275 assertM(fromIntegral lenEncoding + off1 == off2)
256276 assertM(lenArr == off2)
257277 pure val