7cb856b · streamly · git.iko.soy

git.iko.soy/streamly

on master help sign in

Move the Array serialization APIs to the Array module

7cb856bHarendra Kumar3 years ago

benchmark/Streamly/Benchmark/Data/Serialize.hs+1−1

compare with master

unifiedsplit

@@ -24,7 +24,7 @@ import System.Random (randomRIO)

24import Test.QuickCheck (Arbitrary, arbitrary)

25#endif

27−import Streamly.Internal.Data.MutByteArray hiding (encode)

27+import Streamly.Internal.Data.MutByteArray

28import qualified Streamly.Internal.Data.MutByteArray as MBA

29#ifdef USE_UNBOX

30import Data.Proxy (Proxy(..))

core/src/Streamly/Data/MutByteArray.hs+17−12

@@ -5,17 +5,24 @@

5-- Maintainer : streamly@composewell.com

6-- Portability : GHC

7--

8+-- A low level byte Array type for fast binary serialization and

8−-- Fast binary serialization and deserialization of Haskell values to and from

9+-- deserialization of Haskell values. Higher level unboxed array modules

10+-- "Streamly.Data.Array" and "Streamly.Data.MutArray" are built on top of this

9−-- arrays. This module provides two type classes for serialization, 'Unbox' and

11+-- module. This module provides two type classes for serialization, 'Unbox' and

10-- 'Serialize'. The speed is similar to, and in some cases many times faster

12-- 'Serialize'. The speed is similar to, and in some cases many times faster

11-- than the store package. Conceptually, the 'Serialize' type class works in

13-- than the store package. Conceptually, the 'Serialize' type class works in

12-- the same way as store.

14-- the same way as store.

13--

15--

16+-- You probably want the higher level

17+-- 'Streamly.Internal.Data.Array.pinnedSerialize' and

18+-- 'Streamly.Internal.Data.Array.deserialize' from the "Streamly.Data.Array"

19+-- module.

20+--

14-- == Mutable Byte Array

21-- == Mutable Byte Array

15--

22--

16-- 'MutByteArray' is a primitive mutable array in the IO monad. 'Unbox' and

23-- 'MutByteArray' is a primitive mutable array in the IO monad. 'Unbox' and

17-- 'Serialize' type classes use this primitive array to serialize data to and

24-- 'Serialize' type classes use this primitive array to serialize data to and

18−-- deserialize it from. This array can be wrapped into higher level unboxed

25+-- deserialize it from. This array is used to build higher level unboxed

19-- array types 'Streamly.Data.MutArray.MutArray' and 'Streamly.Data.Array.Array'.

26-- array types 'Streamly.Data.MutArray.MutArray' and 'Streamly.Data.Array.Array'.

20--

27--

21-- == Using Unbox

28-- == Using Unbox

@@ -38,9 +45,12 @@

38-- The 'Serialize' type class is a superset of the 'Unbox' type class, it can

45-- The 'Serialize' type class is a superset of the 'Unbox' type class, it can

39-- serialize variable length data types as well e.g. Haskell lists. Use

46-- serialize variable length data types as well e.g. Haskell lists. Use

40-- 'deriveSerialize' to derive the instances of the type class automatically

47-- 'deriveSerialize' to derive the instances of the type class automatically

41−-- and then use 'pinnedEncode', 'decode' to serialize and deserialize the type

48+-- and then use the type class methods to serialize and deserialize to and from

42−-- to and from an 'Array' type. You can also serialize and deserialize directly

49+-- a 'MutByteArray'.

43−-- to and from a 'MutByteArray', using the type class methods.

50+--

51+-- See 'Streamly.Internal.Data.Array.pinnedSerialize' and

52+-- 'Streamly.Internal.Data.Array.deserialize' for 'Array' type based

53+-- serialization.

44--

54--

45module Streamly.Data.MutByteArray

55module Streamly.Data.MutByteArray

46 (

56 (

@@ -62,7 +72,7 @@ module Streamly.Data.MutByteArray

62 -- * Serialize

72 -- * Serialize

63 , Serialize(..)

73 , Serialize(..)

65− -- Deriving instances

75+ -- Deriving Serialize

66 , SerializeConfig

76 , SerializeConfig

67 , serializeConfig

77 , serializeConfig

68 , inlineSize

78 , inlineSize

@@ -71,11 +81,6 @@ module Streamly.Data.MutByteArray

72 , deriveSerialize

82 , deriveSerialize

73 , deriveSerializeWith

83 , deriveSerializeWith

74−

75− -- Encoding and Decoding

76− -- , encode

77− , pinnedEncode

78− , decode

79 ) where

84 ) where

81--------------------------------------------------------------------------------

86--------------------------------------------------------------------------------

core/src/Streamly/Internal/Data/Array.hs+46−2

@@ -76,13 +76,19 @@ module Streamly.Internal.Data.Array

76 -- ** Folding

77 , streamFold

78 , fold

79+

80+ -- ** Serialization

81+ , encodeAs

82+ , serialize

83+ , pinnedSerialize

84+ , deserialize

79 )

85 )

80where

86where

88+#include "assert.hs"

82#include "inline.hs"

89#include "inline.hs"

83#include "ArrayMacros.h"

90#include "ArrayMacros.h"

85−import Control.Exception (assert)

86import Control.Monad (when)

92import Control.Monad (when)

87import Control.Monad.IO.Class (MonadIO(..))

93import Control.Monad.IO.Class (MonadIO(..))

88import Data.Functor.Identity (Identity)

94import Data.Functor.Identity (Identity)

@@ -94,6 +100,8 @@ import Foreign.Storable (Storable)

94import Streamly.Internal.Data.Unbox (Unbox(..))

100import Streamly.Internal.Data.Unbox (Unbox(..))

95import Prelude hiding (length, null, last, map, (!!), read, concat)

101import Prelude hiding (length, null, last, map, (!!), read, concat)

102

103+import Streamly.Internal.Data.MutByteArray.Type (PinnedState(..))

104+import Streamly.Internal.Data.Serialize.Type (Serialize)

97import Streamly.Internal.Data.Fold.Type (Fold(..))

105import Streamly.Internal.Data.Fold.Type (Fold(..))

98import Streamly.Internal.Data.Producer.Type (Producer(..))

106import Streamly.Internal.Data.Producer.Type (Producer(..))

99import Streamly.Internal.Data.Stream (Stream)

107import Streamly.Internal.Data.Stream (Stream)

@@ -101,7 +109,8 @@ import Streamly.Internal.Data.Tuple.Strict (Tuple3Fused'(..))

101import Streamly.Internal.Data.Unfold.Type (Unfold(..))

109import Streamly.Internal.Data.Unfold.Type (Unfold(..))

102import Streamly.Internal.System.IO (unsafeInlineIO)

110import Streamly.Internal.System.IO (unsafeInlineIO)

103

111

104−import qualified Streamly.Internal.Data.MutArray.Type as MA

112+import qualified Streamly.Internal.Data.Serialize.Type as Serialize

113+import qualified Streamly.Internal.Data.MutByteArray.Type as MBA

105import qualified Streamly.Internal.Data.MutArray as MA

114import qualified Streamly.Internal.Data.MutArray as MA

106import qualified Streamly.Internal.Data.Array.Type as A

115import qualified Streamly.Internal.Data.Array.Type as A

107import qualified Streamly.Internal.Data.Fold as FL

116import qualified Streamly.Internal.Data.Fold as FL

@@ -543,3 +552,38 @@ fold f arr = Stream.fold f (A.read arr)

543{-# INLINE streamFold #-}

552{-# INLINE streamFold #-}

544streamFold :: (Monad m, Unbox a) => (Stream m a -> m b) -> Array a -> m b

553streamFold :: (Monad m, Unbox a) => (Stream m a -> m b) -> Array a -> m b

545streamFold f arr = f (A.read arr)

554streamFold f arr = f (A.read arr)

555+

556+--------------------------------------------------------------------------------

557+-- Serialization

558+--------------------------------------------------------------------------------

559+

560+{-# INLINE encodeAs #-}

561+encodeAs :: forall a. Serialize a => PinnedState -> a -> Array Word8

562+encodeAs ps a =

563+ unsafeInlineIO $ do

564+ let len = Serialize.size 0 a

565+ mbarr <- MBA.newBytesAs ps len

566+ off <- Serialize.serialize 0 mbarr a

567+ assertM(len == off)

568+ pure $ Array mbarr 0 off

569+

570+{-# INLINE serialize #-}

571+serialize :: Serialize a => a -> Array Word8

572+serialize = encodeAs Unpinned

573+

574+-- | Serialize a Haskell type to a pinned byte array. The array is allocated

575+-- using pinned memory so that it can be used directly in OS APIs for writing

576+-- to file or sending over the network.

577+{-# INLINE pinnedSerialize #-}

578+pinnedSerialize :: Serialize a => a -> Array Word8

579+pinnedSerialize = encodeAs Pinned

580+

581+-- | Decode a Haskell type from a byte array containing its serialized

582+-- representation.

583+{-# INLINE deserialize #-}

584+deserialize :: Serialize a => Array Word8 -> a

585+deserialize arr@(Array {..}) = unsafeInlineIO $ do

586+ let lenArr = length arr

587+ (off, val) <- Serialize.deserialize arrStart arrContents (arrStart + lenArr)

588+ assertM(off == arrStart + lenArr)

589+ pure val

core/src/Streamly/Internal/Data/Serialize/Type.hs+2−49

@@ -9,36 +9,24 @@

9module Streamly.Internal.Data.Serialize.Type

10 (

11 Serialize(..)

12− , encode

13− , encodeAs

14− , pinnedEncode

15− , decode

16 ) where

12 ) where

18--------------------------------------------------------------------------------

14--------------------------------------------------------------------------------

19-- Imports

15-- Imports

20--------------------------------------------------------------------------------

16--------------------------------------------------------------------------------

22−#include "assert.hs"

23−

24−#ifdef DEBUG

25−import Control.Exception (assert)

26−#endif

27−

28import Data.List (foldl')

18import Data.List (foldl')

29import Data.Proxy (Proxy (..))

19import Data.Proxy (Proxy (..))

30import Streamly.Internal.Data.Unbox (Unbox)

20import Streamly.Internal.Data.Unbox (Unbox)

31−import Streamly.Internal.Data.MutByteArray.Type

21+import Streamly.Internal.Data.MutByteArray.Type (MutByteArray(..))

32− (MutByteArray(..), PinnedState(..))

33import Streamly.Internal.Data.Array.Type (Array(..))

22import Streamly.Internal.Data.Array.Type (Array(..))

34−import Streamly.Internal.System.IO (unsafeInlineIO)

35import GHC.Int (Int16(..), Int32(..), Int64(..), Int8(..))

23import GHC.Int (Int16(..), Int32(..), Int64(..), Int8(..))

36import GHC.Word (Word16(..), Word32(..), Word64(..), Word8(..))

24import GHC.Word (Word16(..), Word32(..), Word64(..), Word8(..))

37import GHC.Stable (StablePtr(..))

25import GHC.Stable (StablePtr(..))

39import qualified Streamly.Internal.Data.MutByteArray.Type as MBA

27import qualified Streamly.Internal.Data.MutByteArray.Type as MBA

40import qualified Streamly.Internal.Data.Unbox as Unbox

28import qualified Streamly.Internal.Data.Unbox as Unbox

41−import qualified Streamly.Internal.Data.Array as Array

29+import qualified Streamly.Internal.Data.Array.Type as Array

42import qualified Streamly.Internal.Data.MutArray as MutArray

30import qualified Streamly.Internal.Data.MutArray as MutArray

44import GHC.Exts

32import GHC.Exts

@@ -314,38 +302,3 @@ instance (Serialize a, Serialize b) => Serialize (a, b) where

314 (off1, a) <- deserialize off arr end

302 (off1, a) <- deserialize off arr end

315 (off2, b) <- deserialize off1 arr end

303 (off2, b) <- deserialize off1 arr end

316 pure (off2, (a, b))

304 pure (off2, (a, b))

317−

318−--------------------------------------------------------------------------------

319−-- High level functions

320−--------------------------------------------------------------------------------

321−

322−{-# INLINE encodeAs #-}

323−encodeAs :: forall a. Serialize a => PinnedState -> a -> Array Word8

324−encodeAs ps a =

325− unsafeInlineIO $ do

326− let len = size 0 a

327− mbarr <- MBA.newBytesAs ps len

328− off <- serialize 0 mbarr a

329− assertM(len == off)

330− pure $ Array mbarr 0 off

331−

332−{-# INLINE encode #-}

333−encode :: Serialize a => a -> Array Word8

334−encode = encodeAs Unpinned

335−

336−-- | Encode a Haskell type to a byte array. The array is allocated using pinned

337−-- memory so that it can be used directly in OS APIs for writing to file or

338−-- sending over the network.

339−{-# INLINE pinnedEncode #-}

340−pinnedEncode :: Serialize a => a -> Array Word8

341−pinnedEncode = encodeAs Pinned

342−

343−-- | Decode a Haskell type from its serialized representation in a byte

344−-- array.

345−{-# INLINE decode #-}

346−decode :: Serialize a => Array Word8 -> a

347−decode arr@(Array {..}) = unsafeInlineIO $ do

348− let lenArr = Array.length arr

349− (off, val) <- deserialize arrStart arrContents (arrStart + lenArr)

350− assertM(off == arrStart + lenArr)

351− pure val

test/Streamly/Test/Data/Serialize.hs+3−3

@@ -185,9 +185,9 @@ peekAndVerify (arr, serStartOff, serEndOff) val = do

185 val2 `shouldBe` val

186 off2 `shouldBe` serEndOff

187 let slice = Array.Array arr serStartOff serEndOff

188− val `shouldBe` Serialize.decode slice

188+ val `shouldBe` Array.deserialize slice

189 clonedSlice <- Array.clone slice

190− val `shouldBe` Serialize.decode clonedSlice

190+ val `shouldBe` Array.deserialize clonedSlice

191

192roundtrip

193 :: forall a. (Eq a, Show a, Serialize.Serialize a)

@@ -195,7 +195,7 @@ roundtrip

195 -> IO ()

196roundtrip val = do

197

198− val `shouldBe` Serialize.decode (Serialize.encode val)

198+ val `shouldBe` Array.deserialize (Array.pinnedSerialize val)

199

200 res <- poke val

201 peekAndVerify res val