git.iko.soy/streamly
on master help sign in
Move the Array serialization APIs to the Array module
commit7cb856b7e66e8a51c53d20f85080c53c8910269f authorHarendra Kumar <harendra@composewell.com> dateSat 25 Nov 2023, 10:19 (3 years ago) parent56fd008
5 files changed+69−67
compare with master
unifiedsplit
benchmark/Streamly/Benchmark/Data/Serialize.hs+1−1 core/src/Streamly/Data/MutByteArray.hs+17−12 core/src/Streamly/Internal/Data/Array.hs+46−2 core/src/Streamly/Internal/Data/Serialize/Type.hs+2−49 test/Streamly/Test/Data/Serialize.hs+3−3
benchmark/Streamly/Benchmark/Data/Serialize.hs+1−1
@@ -24,7 +24,7 @@ import System.Random (randomRIO)
2424import Test.QuickCheck (Arbitrary, arbitrary)
2525#endif
2626
27−import Streamly.Internal.Data.MutByteArray hiding (encode)
27+import Streamly.Internal.Data.MutByteArray
2828import qualified Streamly.Internal.Data.MutByteArray as MBA
2929#ifdef USE_UNBOX
3030import Data.Proxy (Proxy(..))
core/src/Streamly/Data/MutByteArray.hs+17−12
@@ -5,17 +5,24 @@
55-- Maintainer : streamly@composewell.com
66-- Portability : GHC
77--
8−-- Fast binary serialization and deserialization of Haskell values to and from
9−-- arrays. This module provides two type classes for serialization, 'Unbox' and
8+-- A low level byte Array type for fast binary serialization and
9+-- deserialization of Haskell values. Higher level unboxed array modules
10+-- "Streamly.Data.Array" and "Streamly.Data.MutArray" are built on top of this
11+-- module. This module provides two type classes for serialization, 'Unbox' and
1012-- 'Serialize'. The speed is similar to, and in some cases many times faster
1113-- than the store package. Conceptually, the 'Serialize' type class works in
1214-- the same way as store.
1315--
16+-- You probably want the higher level
17+-- 'Streamly.Internal.Data.Array.pinnedSerialize' and
18+-- 'Streamly.Internal.Data.Array.deserialize' from the "Streamly.Data.Array"
19+-- module.
20+--
1421-- == Mutable Byte Array
1522--
1623-- 'MutByteArray' is a primitive mutable array in the IO monad. 'Unbox' and
1724-- 'Serialize' type classes use this primitive array to serialize data to and
18−-- deserialize it from. This array can be wrapped into higher level unboxed
25+-- deserialize it from. This array is used to build higher level unboxed
1926-- array types 'Streamly.Data.MutArray.MutArray' and 'Streamly.Data.Array.Array'.
2027--
2128-- == Using Unbox
@@ -38,9 +45,12 @@
3845-- The 'Serialize' type class is a superset of the 'Unbox' type class, it can
3946-- serialize variable length data types as well e.g. Haskell lists. Use
4047-- 'deriveSerialize' to derive the instances of the type class automatically
41−-- and then use 'pinnedEncode', 'decode' to serialize and deserialize the type
42−-- to and from an 'Array' type. You can also serialize and deserialize directly
43−-- to and from a 'MutByteArray', using the type class methods.
48+-- and then use the type class methods to serialize and deserialize to and from
49+-- a 'MutByteArray'.
50+--
51+-- See 'Streamly.Internal.Data.Array.pinnedSerialize' and
52+-- 'Streamly.Internal.Data.Array.deserialize' for 'Array' type based
53+-- serialization.
4454--
4555module Streamly.Data.MutByteArray
4656 (
@@ -62,7 +72,7 @@ module Streamly.Data.MutByteArray
6272 -- * Serialize
6373 , Serialize(..)
6474
65− -- Deriving instances
75+ -- Deriving Serialize
6676 , SerializeConfig
6777 , serializeConfig
6878 , inlineSize
@@ -71,11 +81,6 @@ module Streamly.Data.MutByteArray
7181
7282 , deriveSerialize
7383 , deriveSerializeWith
74−
75− -- Encoding and Decoding
76− -- , encode
77− , pinnedEncode
78− , decode
7984 ) where
8085
8186--------------------------------------------------------------------------------
core/src/Streamly/Internal/Data/Array.hs+46−2
@@ -76,13 +76,19 @@ module Streamly.Internal.Data.Array
7676 -- ** Folding
7777 , streamFold
7878 , fold
79+
80+ -- ** Serialization
81+ , encodeAs
82+ , serialize
83+ , pinnedSerialize
84+ , deserialize
7985 )
8086where
8187
88+#include "assert.hs"
8289#include "inline.hs"
8390#include "ArrayMacros.h"
8491
85−import Control.Exception (assert)
8692import Control.Monad (when)
8793import Control.Monad.IO.Class (MonadIO(..))
8894import Data.Functor.Identity (Identity)
@@ -94,6 +100,8 @@ import Foreign.Storable (Storable)
94100import Streamly.Internal.Data.Unbox (Unbox(..))
95101import Prelude hiding (length, null, last, map, (!!), read, concat)
96102
103+import Streamly.Internal.Data.MutByteArray.Type (PinnedState(..))
104+import Streamly.Internal.Data.Serialize.Type (Serialize)
97105import Streamly.Internal.Data.Fold.Type (Fold(..))
98106import Streamly.Internal.Data.Producer.Type (Producer(..))
99107import Streamly.Internal.Data.Stream (Stream)
@@ -101,7 +109,8 @@ import Streamly.Internal.Data.Tuple.Strict (Tuple3Fused'(..))
101109import Streamly.Internal.Data.Unfold.Type (Unfold(..))
102110import Streamly.Internal.System.IO (unsafeInlineIO)
103111
104−import qualified Streamly.Internal.Data.MutArray.Type as MA
112+import qualified Streamly.Internal.Data.Serialize.Type as Serialize
113+import qualified Streamly.Internal.Data.MutByteArray.Type as MBA
105114import qualified Streamly.Internal.Data.MutArray as MA
106115import qualified Streamly.Internal.Data.Array.Type as A
107116import qualified Streamly.Internal.Data.Fold as FL
@@ -543,3 +552,38 @@ fold f arr = Stream.fold f (A.read arr)
543552{-# INLINE streamFold #-}
544553streamFold :: (Monad m, Unbox a) => (Stream m a -> m b) -> Array a -> m b
545554streamFold f arr = f (A.read arr)
555+
556+--------------------------------------------------------------------------------
557+-- Serialization
558+--------------------------------------------------------------------------------
559+
560+{-# INLINE encodeAs #-}
561+encodeAs :: forall a. Serialize a => PinnedState -> a -> Array Word8
562+encodeAs ps a =
563+ unsafeInlineIO $ do
564+ let len = Serialize.size 0 a
565+ mbarr <- MBA.newBytesAs ps len
566+ off <- Serialize.serialize 0 mbarr a
567+ assertM(len == off)
568+ pure $ Array mbarr 0 off
569+
570+{-# INLINE serialize #-}
571+serialize :: Serialize a => a -> Array Word8
572+serialize = encodeAs Unpinned
573+
574+-- | Serialize a Haskell type to a pinned byte array. The array is allocated
575+-- using pinned memory so that it can be used directly in OS APIs for writing
576+-- to file or sending over the network.
577+{-# INLINE pinnedSerialize #-}
578+pinnedSerialize :: Serialize a => a -> Array Word8
579+pinnedSerialize = encodeAs Pinned
580+
581+-- | Decode a Haskell type from a byte array containing its serialized
582+-- representation.
583+{-# INLINE deserialize #-}
584+deserialize :: Serialize a => Array Word8 -> a
585+deserialize arr@(Array {..}) = unsafeInlineIO $ do
586+ let lenArr = length arr
587+ (off, val) <- Serialize.deserialize arrStart arrContents (arrStart + lenArr)
588+ assertM(off == arrStart + lenArr)
589+ pure val
core/src/Streamly/Internal/Data/Serialize/Type.hs+2−49
@@ -9,36 +9,24 @@
99module Streamly.Internal.Data.Serialize.Type
1010 (
1111 Serialize(..)
12− , encode
13− , encodeAs
14− , pinnedEncode
15− , decode
1612 ) where
1713
1814--------------------------------------------------------------------------------
1915-- Imports
2016--------------------------------------------------------------------------------
2117
22−#include "assert.hs"
23−
24−#ifdef DEBUG
25−import Control.Exception (assert)
26−#endif
27−
2818import Data.List (foldl')
2919import Data.Proxy (Proxy (..))
3020import Streamly.Internal.Data.Unbox (Unbox)
31−import Streamly.Internal.Data.MutByteArray.Type
32− (MutByteArray(..), PinnedState(..))
21+import Streamly.Internal.Data.MutByteArray.Type (MutByteArray(..))
3322import Streamly.Internal.Data.Array.Type (Array(..))
34−import Streamly.Internal.System.IO (unsafeInlineIO)
3523import GHC.Int (Int16(..), Int32(..), Int64(..), Int8(..))
3624import GHC.Word (Word16(..), Word32(..), Word64(..), Word8(..))
3725import GHC.Stable (StablePtr(..))
3826
3927import qualified Streamly.Internal.Data.MutByteArray.Type as MBA
4028import qualified Streamly.Internal.Data.Unbox as Unbox
41−import qualified Streamly.Internal.Data.Array as Array
29+import qualified Streamly.Internal.Data.Array.Type as Array
4230import qualified Streamly.Internal.Data.MutArray as MutArray
4331
4432import GHC.Exts
@@ -314,38 +302,3 @@ instance (Serialize a, Serialize b) => Serialize (a, b) where
314302 (off1, a) <- deserialize off arr end
315303 (off2, b) <- deserialize off1 arr end
316304 pure (off2, (a, b))
317−
318−--------------------------------------------------------------------------------
319−-- High level functions
320−--------------------------------------------------------------------------------
321−
322−{-# INLINE encodeAs #-}
323−encodeAs :: forall a. Serialize a => PinnedState -> a -> Array Word8
324−encodeAs ps a =
325− unsafeInlineIO $ do
326− let len = size 0 a
327− mbarr <- MBA.newBytesAs ps len
328− off <- serialize 0 mbarr a
329− assertM(len == off)
330− pure $ Array mbarr 0 off
331−
332−{-# INLINE encode #-}
333−encode :: Serialize a => a -> Array Word8
334−encode = encodeAs Unpinned
335−
336−-- | Encode a Haskell type to a byte array. The array is allocated using pinned
337−-- memory so that it can be used directly in OS APIs for writing to file or
338−-- sending over the network.
339−{-# INLINE pinnedEncode #-}
340−pinnedEncode :: Serialize a => a -> Array Word8
341−pinnedEncode = encodeAs Pinned
342−
343−-- | Decode a Haskell type from its serialized representation in a byte
344−-- array.
345−{-# INLINE decode #-}
346−decode :: Serialize a => Array Word8 -> a
347−decode arr@(Array {..}) = unsafeInlineIO $ do
348− let lenArr = Array.length arr
349− (off, val) <- deserialize arrStart arrContents (arrStart + lenArr)
350− assertM(off == arrStart + lenArr)
351− pure val
test/Streamly/Test/Data/Serialize.hs+3−3
@@ -185,9 +185,9 @@ peekAndVerify (arr, serStartOff, serEndOff) val = do
185185 val2 `shouldBe` val
186186 off2 `shouldBe` serEndOff
187187 let slice = Array.Array arr serStartOff serEndOff
188− val `shouldBe` Serialize.decode slice
188+ val `shouldBe` Array.deserialize slice
189189 clonedSlice <- Array.clone slice
190− val `shouldBe` Serialize.decode clonedSlice
190+ val `shouldBe` Array.deserialize clonedSlice
191191
192192roundtrip
193193 :: forall a. (Eq a, Show a, Serialize.Serialize a)
@@ -195,7 +195,7 @@ roundtrip
195195 -> IO ()
196196roundtrip val = do
197197
198− val `shouldBe` Serialize.decode (Serialize.encode val)
198+ val `shouldBe` Array.deserialize (Array.pinnedSerialize val)
199199
200200 res <- poke val
201201 peekAndVerify res val