e7bb4d0 · streamly · git.iko.soy

git.iko.soy/streamly

Update the documentation of Serialize and Unbox

commite7bb4d0ed8178191443fd3ce1579feb26e129bfc authorHarendra Kumar <harendra@composewell.com> dateTue 21 Nov 2023, 07:03 (3 years ago) parent9f04238

5 files changed+187−104

compare with master

unifiedsplit

core/src/Streamly/Data/Serialize.hs+36−17 core/src/Streamly/Internal/Data/Serialize/TH.hs+19−11 core/src/Streamly/Internal/Data/Serialize/Type.hs+57−21 core/src/Streamly/Internal/Data/Unbox.hs+59−45 core/src/Streamly/Internal/Data/Unbox/TH.hs+16−10

core/src/Streamly/Data/Serialize.hs+36−17

@@ -5,10 +5,43 @@

55-- Maintainer : streamly@composewell.com

66-- Portability : GHC

77--

8+-- Fast binary serialization and deserialization of Haskell values to and from

9+-- arrays. This module provides two type classes for serialization, 'Unbox' and

10+-- 'Serialize'. The speed is similar to, and in some cases many times faster

11+-- than the store package. Conceptually, the 'Serialize' type class works in

12+-- the same way as store.

13+--

14+-- == Using Unbox

15+--

16+-- The 'Unbox' type class is simple and used to serialize non-recursive fixed

17+-- size data types. This type class is primarily used to implement unboxed

18+-- arrays. Unboxed arrays are just serialization of fixed length Haskell data

19+-- types. Instances of this type class can be derived using 'Generic' or

20+-- template haskell based deriving functions provided in this module. Read the

21+-- type class documentation for more details.

22+--

23+-- Writing a data type to an array using the array creation routines in

24+-- "Streamly.Data.Array" or "Streamly.Data.MutArray", serializes the type to

25+-- the array. Similarly, reading the data type from the array deserializes it.

26+--

27+-- There are no encode/decode routines provided to serialize or deserialize

28+-- using this type class but you can write those easily. Just create a mutable

29+-- byte array and use 'pokeByteIndex' to serialize a type, and use

30+-- 'peekByteIndex' to deserialize it.

31+--

32+-- == Using Serialize

33+--

34+-- The 'Serialize' type class is a superset of the 'Unbox' type class, it can

35+-- serialize variable length data types as well e.g. Haskell lists. Use

36+-- 'deriveSerialize' to derive the instances of the type class automatically

37+-- and then use 'pinnedEncode', 'decode' to serialize and deserialize the type

38+-- respectively. You can also create the array yourself and use the type class

39+-- functions directly.

40+--

841module Streamly.Data.Serialize

942 (

1043

11− -- * Types

44+ -- * MutableByteArray

1245 MutableByteArray

1346

1447 -- * Unbox

@@ -18,14 +51,14 @@ module Streamly.Data.Serialize

1851

1952 -- * Serialize

2053 , Serialize(..)

54+ -- XXX Need to export the config setters as well.

55+ , defaultConfig

2156 , deriveSerialize

2257 , deriveSerializeWith

2358

24− -- * Encoding

2559 -- , encode

2660 , pinnedEncode

2761

28− -- * Decoding

2962 , decode

3063 ) where

3164

@@ -34,17 +67,3 @@ module Streamly.Data.Serialize

3467--------------------------------------------------------------------------------

3568

3669import Streamly.Internal.Data.Serialize

37− ( Serialize(..)

38− , decode

39− -- , encode

40− , pinnedEncode

41− )

42−import Streamly.Internal.Data.Serialize.TH

43− ( deriveSerialize

44− , deriveSerializeWith

45− )

46−import Streamly.Internal.Data.Unbox

47− ( MutableByteArray

48− , Unbox(..)

49− )

50−import Streamly.Internal.Data.Unbox.TH (deriveUnbox, deriveUnboxWith)

core/src/Streamly/Internal/Data/Serialize/TH.hs+19−11

@@ -426,13 +426,18 @@ deriveSerializeInternal conf headTy cons next = do

426426 let methods = concat [sizeDec, peekDec, pokeDec]

427427 next methods

428428

429−-- | Similar to 'deriveSerialize' but take a 'Config' to control how the

430−-- instance is generated.

429+-- | @deriveSerializeWith config instance-dec@ generates a template Haskell

430+-- splice consisting of a declaration of a 'Serialize' instance. @instance-dec@

431+-- is a template Haskell declaration splice consisting of a standard Haskell

432+-- instance declaration without the type class methods. The type class methods

433+-- for the given instance are generated according to the supplied @config@

434+-- parameter.

435+--

436+-- For example:

431437--

432−-- Usage:

433438-- @

434−-- $(deriveSerializeWith

435−-- config

439+-- \$(deriveSerializeWith

440+-- defaultConfig

436441-- [d|instance Serialize a => Serialize (Maybe a)|])

437442-- @

438443deriveSerializeWith :: Config -> Q [Dec] -> Q [Dec]

@@ -478,23 +483,26 @@ deriveSerializeWith conf mDecs = do

478483 go _ = errorMessage dec

479484

480485

481−-- | Template haskell helper to create instances of 'Serialize' automatically.

486+-- | @deriveSerialize dataTypeName@ generates a template Haskell splice

487+-- consisting of a declaration of the 'Serialize' instance for the given

488+-- dataTypeName, all type parameters of dataTypeName are required to have the

489+-- 'Serialize' constraint.

490+--

491+-- For example,

482492--

483−-- Consider the datatype:

484493-- @

485494-- data CustomDataType a b c = ...

495+-- \$(deriveSerialize ''CustomDataType)

486496-- @

487497--

488−-- Usage: @$(deriveSerialize ''CustomDataType)@

498+-- Generates the following code:

489499--

490−-- Note: All type variables automatcally get an "Serialize" constraint.

491−-- The derived code will look like the following,

492500-- @

493501-- instance (Serialize a, Serialize b, Serialize c) => Serialize (CustomDataType a b c) where

494502-- ...

495503-- @

496504--

497−-- To control which type variables don't get the Serialize constraint, use

505+-- To control which type parameters get the Serialize constraint, use

498506-- 'deriveSerializeWith'.

499507deriveSerialize :: Name -> Q [Dec]

500508deriveSerialize name = do

core/src/Streamly/Internal/Data/Serialize/Type.hs+57−21

@@ -61,21 +61,33 @@ import GHC.Exts

6161-- Types

6262--------------------------------------------------------------------------------

6363

64−-- | A type implementing the 'Serialize' interface supplies operations for

65−-- reading and writing the type from and to a mutable byte array (an unboxed

66−-- representation of the type) in memory. The read operation 'deserialize'

67−-- deserializes the boxed type from the mutable byte array. The write operation

68−-- 'serialize' serializes the boxed type to the mutable byte array.

64+-- | The 'Serialize' type class provides operations for serialization and

65+-- deserialization of general Haskell data types to and from their byte stream

66+-- representation.

6967--

70−-- IMPORTANT: The serialized data's byte ordering is not normalized, which means

71−-- it remains the same as the machine's byte order where the function is

72−-- executed. Consequently, it may not be compatible across machines with

73−-- different byte ordering.

68+-- Unlike 'Unbox', 'Serialize' uses variable length encoding, therefore, it can

69+-- serialize recursive and variable length data types like lists, or variable

70+-- length sum types where the length of the value may vary depending on a

71+-- particular constructor. For variable length data types the length is encoded

72+-- along with the data.

7473--

75−-- 'Serialize' contains enough information to serialize and deserialize variable

76−-- length types.

74+-- The 'deserialize' operation reads bytes from the mutable byte array and

75+-- builds a Haskell data type from these bytes, the number of bytes it reads

76+-- depends on the type and the encoded value it is reading. 'serialize'

77+-- operation converts a Haskell data type to its binary representation which

78+-- must consist of as many bytes as returned by the @size@ operation on that

79+-- value and then stores these bytes into the mutable byte array. The

80+-- programmer is expected to use the @size@ operation and allocate an array of

81+-- sufficient length before calling 'serialize'.

7782--

78−-- >>> import Streamly.Internal.Data.Serialize.Type (Serialize(..))

83+-- IMPORTANT: The serialized data's byte ordering remains the same as the host

84+-- machine's byte order. Therefore, it can not be deserialized from host

85+-- machines with a different byte ordering.

86+--

87+-- Instances can be derived via Template Haskell, or written manually.

88+--

89+-- Here is an example, for deriving an instance of this type class using

90+-- template Haskell:

7991--

8092-- >>> :{

8193-- data Object = Object

@@ -84,10 +96,24 @@ import GHC.Exts

8496-- }

8597-- :}

8698--

99+-- @

100+-- import Streamly.Data.Serialize (deriveSerialize)

101+-- \$(deriveSerialize ''Object)

102+-- @

103+--

104+-- See 'Streamly.Data.Serialize.deriveSerialize' and

105+-- 'Streamly.Data.Serialize.deriveSerializeWith' for more information on

106+-- deriving using Template Haskell.

107+--

108+-- Here is an example of a manual instance.

109+--

110+-- >>> import Streamly.Data.Serialize (Serialize(..))

111+--

87112-- >>> :{

88113-- instance Serialize Object where

89114-- size acc obj = size (size acc (_obj1 obj)) (_obj2 obj)

90115-- deserialize i arr len = do

116+-- -- Check the array bounds before reading

91117-- (i1, x0) <- deserialize i arr len

92118-- (i2, x1) <- deserialize i1 arr len

93119-- pure (i2, Object x0 x1)

@@ -104,23 +130,28 @@ class Serialize a where

104130 -- It is of the form @Int -> a -> Int@ because you can have tail-recursive

105131 -- traversal of the structures.

106132

107− -- | Get the size, in bytes, reqired to store the serialized

108− -- representation of the type. Size cannot be zero.

133+ -- | @size accum value@ returns @accum@ incremented by the size of the

134+ -- serialized representation of @value@ in bytes. Size cannot be zero. It

135+ -- should be at least 1 byte.

109136 size :: Int -> a -> Int

110137

111138 -- We can implement the following functions without returning the `Int`

112139 -- offset but that may require traversing the Haskell structure again to get

113140 -- the size. Therefore, this is a performance optimization.

114141

115− -- | @deserialize offset array arrayLen@ deserializes a value from the

116− -- given byte-index in the array. Returns a tuple of the next byte-index

117− -- and the deserialized value.

142+ -- | @deserialize byte-offset array arrayLen@ deserializes a value from the

143+ -- given byte-offset in the array. Returns a tuple consisting of the next

144+ -- byte-offset and the deserialized value.

145+ --

146+ -- Throws an exception if the operation would exceed the supplied arrayLen.

118147 deserialize :: Int -> MutableByteArray -> Int -> IO (Int, a)

119148

120− -- | Write the serialized representation of the value in the array at the

121− -- given byte-index. Returns the next byte-index. This is an unsafe

122− -- operation, the programmer must ensure that the array has enough space

123− -- available in the array to serialize the value as determined by the

149+ -- | @serialize byte-offset array value@ writes the serialized

150+ -- representation of the @value@ in the array at the given byte-offset.

151+ -- Returns the next byte-offset.

152+ --

153+ -- This is an unsafe operation, the programmer must ensure that the array

154+ -- has enough space available to serialize the value as determined by the

124155 -- @size@ operation.

125156 serialize :: Int -> MutableByteArray -> a -> IO Int

126157

@@ -303,10 +334,15 @@ encodeAs ps a =

303334encode :: Serialize a => a -> Array Word8

304335encode = encodeAs Unpinned

305336

337+-- | Encode a Haskell type to a byte array. The array is allocated using pinned

338+-- memory so that it can be used directly in OS APIs for writing to file or

339+-- sending over the network.

306340{-# INLINE pinnedEncode #-}

307341pinnedEncode :: Serialize a => a -> Array Word8

308342pinnedEncode = encodeAs Pinned

309343

344+-- | Decode a Haskell type from its serialized representation in a byte

345+-- array.

310346{-# INLINE decode #-}

311347decode :: Serialize a => Array Word8 -> a

312348decode arr@(Array {..}) = unsafeInlineIO $ do

core/src/Streamly/Internal/Data/Unbox.hs+59−45

@@ -12,12 +12,17 @@

1212-- Portability : GHC

1313--

1414module Streamly.Internal.Data.Unbox

15− ( Unbox(..)

16− , PinnedState(..)

17− , MutableByteArray(..)

18− , touch

15+ (

16+ -- ** MutableByteArray

17+ MutableByteArray(..)

1918 , getMutableByteArray#

2019 , sizeOfMutableByteArray

20+ , touch

21+ , nil

22+ , putSliceUnsafe

23+

24+ -- ** Allocation

25+ , PinnedState(..)

2126 , isPinned

2227 , pin

2328 , unpin

@@ -25,30 +30,29 @@ module Streamly.Internal.Data.Unbox

2530 , newBytes

2631 , pinnedNewBytes

2732 , pinnedNewAlignedBytes

28− , nil

29− , putSliceUnsafe

3033

31− -- * Type Parser and Builder

32− , BoundedPtr (..)

34+ -- ** Unbox type class

35+ , Unbox(..)

3336

37+ -- ** Peek and poke utilities

38+ , BoundedPtr (..)

39+ -- Peek

3440 , Peeker (..)

3541 , read

3642 , readUnsafe

3743 , skipByte

3844 , runPeeker

39−

45+ -- Poke

4046 , pokeBoundedPtrUnsafe

4147 , pokeBoundedPtr

4248

43− -- * Generic Unbox instances

44− , genericSizeOf

45− , genericPeekByteIndex

46− , genericPokeByteIndex

47−

48− -- Classess used for generic deriving.

49+ -- ** Generic Deriving

4950 , PeekRep(..)

5051 , PokeRep(..)

5152 , SizeOfRep(..)

53+ , genericSizeOf

54+ , genericPeekByteIndex

55+ , genericPokeByteIndex

5256 ) where

5357

5458#include "MachDeps.h"

@@ -92,6 +96,10 @@ data PinnedState

9296 | Unpinned

9397

9498-- XXX can use UnliftedNewtypes

99+

100+-- | A lifted mutable byte array type wrapping @MutableByteArray# RealWorld@.

101+-- This is a low level array used to back high level unboxed arrays and

102+-- serialized data.

95103data MutableByteArray = MutableByteArray (MutableByteArray# RealWorld)

96104

97105{-# INLINE getMutableByteArray# #-}

@@ -327,23 +335,37 @@ unpin arr@(MutableByteArray marr#) =

327335-- size fields at each nesting level, aggregating the size upwards.

328336

329337-- | The 'Unbox' type class provides operations for serialization (unboxing)

330−-- and deserialization (boxing) of a fixed-length, non-recursive Haskell data

331−-- type to and from its unboxed byte representation in a mutable byte array.

338+-- and deserialization (boxing) of fixed-length, non-recursive Haskell data

339+-- types to and from their byte stream representation.

340+--

332341-- Unbox uses fixed size encoding, therefore, size is independent of the value,

333−-- it can be determined solely by the type. This makes types with 'Unbox'

334−-- instances suitable for storing in arrays. Note that sum types may have

335−-- multiple constructors of different sizes, the size of the sum type is

342+-- it must be determined solely by the type. This restriction makes types with

343+-- 'Unbox' instances suitable for storing in arrays. Note that sum types may

344+-- have multiple constructors of different sizes, the size of a sum type is

336345-- computed as the maximum required by any constructor.

337346--

338−-- The 'peekByteIndex' read operation converts a Haskell type from its unboxed

339−-- representation stored in a mutable byte array, while the 'pokeByteIndex'

340−-- write operation serializes a Haskell data type to its byte representation in

341−-- the mutable byte array. These operations do not check the bounds of the

342−-- array, the user of the type class is expected to check the bounds before

343−-- peeking or poking.

347+-- The 'peekByteIndex' operation reads as many bytes from the mutable byte

348+-- array as the @size@ of the data type and builds a Haskell data type from

349+-- these bytes. 'pokeByteIndex' operation converts a Haskell data type to its

350+-- binary representation which consists of @size@ bytes and then stores

351+-- these bytes into the mutable byte array. These operations do not check the

352+-- bounds of the array, the user of the type class is expected to check the

353+-- bounds before peeking or poking.

354+--

355+-- IMPORTANT: The serialized data's byte ordering remains the same as the host

356+-- machine's byte order. Therefore, it can not be deserialized from host

357+-- machines with a different byte ordering.

344358--

345−-- Instances can be derived via Generics or Template Haskell. Note that the

346−-- data type must be non-recursive.

359+-- Instances can be derived via Generics, Template Haskell, or written

360+-- manually. Note that the data type must be non-recursive. WARNING! Generic

361+-- and Template Haskell deriving, both hang for recursive data types. Deriving

362+-- via Generics is more convenient but Template Haskell should be preferred

363+-- over Generics for the following reasons:

364+--

365+-- 1. Instances derived via Template Haskell provide better and more reliable

366+-- performance.

367+-- 2. Generic deriving allows only 256 fields or constructor tags whereas

368+-- template Haskell has no limit.

347369--

348370-- Here is an example, for deriving an instance of this type class using

349371-- generics:

@@ -356,28 +378,20 @@ unpin arr@(MutableByteArray marr#) =

356378-- } deriving Generic

357379-- :}

358380--

381+-- >>> import Streamly.Data.Serialize (Unbox(..))

382+-- >>> instance Unbox Object

383+--

359384-- To derive the instance via Template Haskell:

360385--

361386-- @

362387-- import Streamly.Data.Serialize (deriveUnbox)

363−-- $(deriveUnbox ''Object)

388+-- \$(deriveUnbox ''Object)

364389-- @

365390--

366391-- See 'Streamly.Data.Serialize.deriveUnbox' and

367392-- 'Streamly.Data.Serialize.deriveUnboxWith' for more information on deriving

368393-- using Template Haskell.

369394--

370−-- Deriving via Template Haskell should be preferred for the following reasons:

371−-- 1. Instances derived via Template Haskell have more performant routines.

372−-- 2. Template Haskell deriving can handle (maxBound :: Word64) number of

373−-- constructors whereas the Generic deriving can only handle 256.

374−--

375−-- WARNING! Generic and Template Haskell deriving, both hang for recursive data

376−-- types.

377−--

378−-- >>> import Streamly.Data.Array (Unbox(..))

379−-- >>> instance Unbox Object

380−--

381395-- If you want to write the instance manually:

382396--

383397-- >>> :{

@@ -395,15 +409,15 @@ unpin arr@(MutableByteArray marr#) =

395409-- :}

396410--

397411class Unbox a where

398− -- | Get the size. Size cannot be zero.

412+ -- | Get the size. Size cannot be zero, should be at least 1 byte.

399413 sizeOf :: Proxy a -> Int

400414

401415 {-# INLINE sizeOf #-}

402416 default sizeOf :: (SizeOfRep (Rep a)) => Proxy a -> Int

403417 sizeOf = genericSizeOf

404418

405− -- | Read an element of type "a" from a MutableByteArray given the byte

406− -- index.

419+ -- | @peekByteIndex byte-index array@ reads an element of type @a@ from the

420+ -- the given the byte index in the array.

407421 --

408422 -- IMPORTANT: The implementation of this interface may not check the bounds

409423 -- of the array, the caller must not assume that.

@@ -414,8 +428,8 @@ class Unbox a where

414428 Int -> MutableByteArray -> IO a

415429 peekByteIndex i arr = genericPeekByteIndex arr i

416430

417− -- | Write an element of type "a" to a MutableByteArray given the byte

418− -- index.

431+ -- | @pokeByteIndex byte-index array@ writes an element of type @a@ to the

432+ -- the given the byte index in the array.

419433 --

420434 -- IMPORTANT: The implementation of this interface may not check the bounds

421435 -- of the array, the caller must not assume that.

core/src/Streamly/Internal/Data/Unbox/TH.hs+16−10

@@ -430,23 +430,25 @@ deriveUnboxInternal preds headTy cons = do

430430 ]

431431 return [InstanceD Nothing preds (AppT (ConT ''Unbox) headTy) methods]

432432

433−-- | Template haskell helper to create instances of 'Unbox' automatically.

433+-- | @deriveUnbox dataTypeName@ generates a template Haskell splice consisting

434+-- of a declaration of the 'Unbox' instance for the given dataTypeName, all type

435+-- parameters of dataTypeName are required to have the 'Unbox' constraint.

436+--

437+-- For example,

434438--

435−-- Consider the datatype:

436439-- @

437440-- data CustomDataType a b c = ...

441+-- \$(deriveUnbox ''CustomDataType)

438442-- @

439443--

440−-- Usage: @$(deriveUnbox ''CustomDataType)@

444+-- Generates the following code:

441445--

442−-- Note: All type variables automatcally get an "Unbox" constraint.

443−-- The derived code will look like the following,

444446-- @

445447-- instance (Unbox a, Unbox b, Unbox c) => Unbox (CustomDataType a b c) where

446448-- ...

447449-- @

448450--

449−-- To control which type variables get the Unbox constraint, use

451+-- To control which type parameters get the Unbox constraint, use

450452-- 'deriveUnboxWith'.

451453deriveUnbox :: Name -> Q [Dec]

452454deriveUnbox name = do

@@ -465,15 +467,19 @@ deriveUnbox name = do

465467 ClassP ''Unbox [ty]

466468#endif

467469

468−-- | Like 'deriveUnbox' but control which type variables get the 'Unbox'

469−-- constraint.

470+-- | @deriveUnbox dataTypeName@ generates a template Haskell splice consisting

471+-- of a declaration of the 'Unbox' instance for the given dataTypeName,

472+-- varNames is a list of parameter names of dataTypeName that are required

473+-- to have the 'Unbox' constraint.

474+--

475+-- For example,

470476--

471−-- Consider the datatype:

472477-- @

473478-- data CustomDataType a b c = ...

479+-- \$(deriveUnboxWith ["a", "c"] ''CustomDataType)

474480-- @

475481--

476−-- Usage: @$(deriveUnboxWith ["a", "c"] ''CustomDataType)@

482+-- Generates the following code:

477483--

478484-- @

479485-- instance (Unbox a, Unbox c) => Unbox (CustomDataType a b c) where