@@ -31,7 +31,6 @@ import Streamly.Internal.Data.Unbox
31 ( MutableByteArray(..)
31 ( MutableByteArray(..)
32 , PinnedState(..)
32 , PinnedState(..)
34− , sizeOfMutableByteArray
36import Streamly.Internal.Data.Array.Type (Array(..))
35import Streamly.Internal.Data.Array.Type (Array(..))
37import Streamly.Internal.System.IO (unsafeInlineIO)
36import Streamly.Internal.System.IO (unsafeInlineIO)
@@ -83,9 +82,9 @@ newtype Size a = Size (Int -> a -> Int) -- a left fold or Sum monoid
83-- (Size f, Size g) ->
82-- (Size f, Size g) ->
84-- Size $ \acc obj ->
83-- Size $ \acc obj ->
85-- acc + f 0 (_obj1 obj) + g 0 (_obj2 obj)
84-- acc + f 0 (_obj1 obj) + g 0 (_obj2 obj)
86−-- deserialize i arr = do
85+-- deserialize i arr len = do
87−-- (i1, x0) <- deserialize i arr
86+-- (i1, x0) <- deserialize i arr len
88−-- (i2, x1) <- deserialize i1 arr
87+-- (i2, x1) <- deserialize i1 arr len
89-- pure (i2, Object x0 x1)
88-- pure (i2, Object x0 x1)
90-- serialize i arr (Object x0 x1) = do
89-- serialize i arr (Object x0 x1) = do
91-- i1 <- serialize i arr x0
90-- i1 <- serialize i arr x0
@@ -102,12 +101,16 @@ class Serialize a where
102 -- offset but that may require traversing the Haskell structure again to get
101 -- offset but that may require traversing the Haskell structure again to get
103 -- the size. Therefore, this is a performance optimization.
102 -- the size. Therefore, this is a performance optimization.
104+ -- | @deserialize offset array arrayLen@ deserializes a value from the
105− -- | Deserialize a value from the given byte-index in the array. Returns a
105+ -- given byte-index in the array. Returns a tuple of the next byte-index
106− -- tuple of the next byte-index and the deserialized value.
106+ -- and the deserialized value.
107− deserialize :: Int -> MutableByteArray -> IO (Int, a)
107+ deserialize :: Int -> MutableByteArray -> Int -> IO (Int, a)
109 -- | Write the serialized representation of the value in the array at the
109 -- | Write the serialized representation of the value in the array at the
110− -- given byte-index. Returns the next byte-index.
110+ -- given byte-index. Returns the next byte-index. This is an unsafe
111+ -- operation, the programmer must ensure that the array has enough space
112+ -- available in the array to serialize the value as determined by the
113+ -- @size@ operation.
111 serialize :: Int -> MutableByteArray -> a -> IO Int
114 serialize :: Int -> MutableByteArray -> a -> IO Int
113--------------------------------------------------------------------------------
116--------------------------------------------------------------------------------
@@ -132,12 +135,29 @@ checkBounds _label _off _arr = do
132 else return ()
135 else return ()
138+-- Note: Instead of passing around the size parameter, we can use
139+-- (sizeOfMutableByteArray arr) for checking the array bound, but that turns
140+-- out to be more expensive.
142+-- Another way to optimize this is to avoid the check for fixed size
143+-- structures. For fixed size structures we can do a check at the top level and
144+-- then use checkless deserialization using the Unbox type class. That will
145+-- require ConstSize and VarSize constructors in size. The programmer can
146+-- bundle all const size fields in a newtype to make serialization faster. This
147+-- can speed up the computation of size when serializing and checking size when
150+-- For variable size non-recursive structures a separate size validation method
151+-- could be used to validate the size before deserializing. "validate" can also
152+-- be used to collpase multiple chunks of arrays coming from network into a
153+-- single array for deserializing. But that can also be done by framing the
154+-- serialized value with a size header.
135{-# INLINE deserializeUnsafe #-}
156{-# INLINE deserializeUnsafe #-}
136−deserializeUnsafe :: forall a. Unbox a => Int -> MutableByteArray -> IO (Int, a)
157+deserializeUnsafe :: forall a. Unbox a => Int -> MutableByteArray -> Int -> IO (Int, a)
137−deserializeUnsafe off arr =
158+deserializeUnsafe off arr sz =
138 let next = off + Unbox.sizeOf (Proxy :: Proxy a)
159 let next = off + Unbox.sizeOf (Proxy :: Proxy a)
140− sz <- sizeOfMutableByteArray arr
141 -- Keep likely path in the straight branch.
161 -- Keep likely path in the straight branch.
142 if (next <= sz)
162 if (next <= sz)
143 then Unbox.peekByteIndex off arr >>= \val -> pure (next, val)
163 then Unbox.peekByteIndex off arr >>= \val -> pure (next, val)
@@ -162,7 +182,7 @@ instance Serialize _type where \
162; {-# INLINE size #-} \
182; {-# INLINE size #-} \
163; size = Size (\acc _ -> acc + Unbox.sizeOf (Proxy :: Proxy _type)) \
183; size = Size (\acc _ -> acc + Unbox.sizeOf (Proxy :: Proxy _type)) \
164; {-# INLINE deserialize #-} \
184; {-# INLINE deserialize #-} \
165−; deserialize off arr = deserializeUnsafe off arr :: IO (Int, _type) \
185+; deserialize off arr end = deserializeUnsafe off arr end :: IO (Int, _type) \
166; {-# INLINE serialize #-} \
186; {-# INLINE serialize #-} \
167; serialize = \
187; serialize = \
168 serializeUnsafe :: Int -> MutableByteArray -> _type -> IO Int
188 serializeUnsafe :: Int -> MutableByteArray -> _type -> IO Int
@@ -193,19 +213,19 @@ instance forall a. Serialize a => Serialize [a] where
193 Size f -> foldl' f (acc + (Unbox.sizeOf (Proxy :: Proxy Int))) xs
213 Size f -> foldl' f (acc + (Unbox.sizeOf (Proxy :: Proxy Int))) xs
195 {-# INLINE deserialize #-}
215 {-# INLINE deserialize #-}
196− deserialize off arr = do
216+ deserialize off arr sz = do
197 len <- Unbox.peekByteIndex off arr :: IO Int
217 len <- Unbox.peekByteIndex off arr :: IO Int
198 let off1 = off + Unbox.sizeOf (Proxy :: Proxy Int)
218 let off1 = off + Unbox.sizeOf (Proxy :: Proxy Int)
200 peekList f o i | i >= 3 = do
220 peekList f o i | i >= 3 = do
201 -- Unfold the loop three times
221 -- Unfold the loop three times
202− (o1, x1) <- deserialize o arr
222+ (o1, x1) <- deserialize o arr sz
203− (o2, x2) <- deserialize o1 arr
223+ (o2, x2) <- deserialize o1 arr sz
204− (o3, x3) <- deserialize o2 arr
224+ (o3, x3) <- deserialize o2 arr sz
205 peekList (f . (\xs -> x1:x2:x3:xs)) o3 (i - 3)
225 peekList (f . (\xs -> x1:x2:x3:xs)) o3 (i - 3)
206 peekList f o 0 = pure (o, f [])
226 peekList f o 0 = pure (o, f [])
207 peekList f o i = do
227 peekList f o i = do
208− (o1, x) <- deserialize o arr
228+ (o1, x) <- deserialize o arr sz
209 peekList (f . (x:)) o1 (i - 1)
229 peekList (f . (x:)) o1 (i - 1)
210 peekList id off1 len
230 peekList id off1 len
@@ -250,8 +270,8 @@ pinnedEncode = encodeAs Pinned
250decode :: Serialize a => Array Word8 -> a
270decode :: Serialize a => Array Word8 -> a
251decode arr@(Array {..}) = unsafeInlineIO $ do
271decode arr@(Array {..}) = unsafeInlineIO $ do
252 let lenArr = Array.length arr
272 let lenArr = Array.length arr
253− (off1, lenEncoding :: Int64) <- deserialize 0 arrContents
273+ (off1, lenEncoding :: Int64) <- deserialize 0 arrContents lenArr
254− (off2, val) <- deserialize off1 arrContents
274+ (off2, val) <- deserialize off1 arrContents lenArr
255 assertM(fromIntegral lenEncoding + off1 == off2)
275 assertM(fromIntegral lenEncoding + off1 == off2)
256 assertM(lenArr == off2)
276 assertM(lenArr == off2)