1{-# LANGUAGE TemplateHaskell #-}
4-- Module : Streamly.Internal.Data.Serialize.TH
5-- Copyright : (c) 2023 Composewell Technologies
6-- License : BSD3-3-Clause
7-- Maintainer : streamly@composewell.com
8-- Stability : experimental
11module Streamly.Internal.Data.Serialize.TH
18--------------------------------------------------------------------------------
20--------------------------------------------------------------------------------
23import Data.List (foldl')
24import Data.Word (Word16, Word32, Word64, Word8)
25import Data.Bits (Bits, (.|.), shiftL, zeroBits, xor)
26import Streamly.Internal.System.IO (unsafeInlineIO)
28import Language.Haskell.TH
29import Language.Haskell.TH.Syntax
30import Streamly.Internal.Data.Serialize
32import qualified Streamly.Internal.Data.Unbox as Unbox
34import Streamly.Internal.Data.Unbox.TH
42--------------------------------------------------------------------------------
44--------------------------------------------------------------------------------
46type Field = (Maybe Name, Type)
61_initialOffset = mkName "initialOffset"
64_endOffset = mkName "endOffset"
69mkFieldName :: Int -> Name
70mkFieldName i = mkName ("field" ++ show i)
73makeI i = mkName $ "i" ++ show i
76makeN i = mkName $ "n" ++ show i
79makeA i = mkName $ "a" ++ show i
81--------------------------------------------------------------------------------
82-- Domain specific helpers
83--------------------------------------------------------------------------------
85matchConstructor :: Name -> Int -> Q Exp -> Q Match
86matchConstructor cname numFields exp0 =
88 (conP cname (map varP (map mkFieldName [0 .. (numFields - 1)])))
92exprGetSize :: Q Exp -> (Int, Type) -> Q Exp
93exprGetSize acc (i, _) = [|size $(acc) $(varE (mkFieldName i))|]
95getTagSize :: Int -> Int
96getTagSize numConstructors
97 | numConstructors == 1 = 0
98 | fromIntegral (maxBound :: Word8) >= numConstructors = 1
99 | fromIntegral (maxBound :: Word16) >= numConstructors = 2
100 | fromIntegral (maxBound :: Word32) >= numConstructors = 4
101 | fromIntegral (maxBound :: Word64) >= numConstructors = 8
102 | otherwise = error "Too many constructors"
104getTagType :: Int -> Name
105getTagType numConstructors
106 | numConstructors == 1 = error "No tag for 1 constructor"
107 | fromIntegral (maxBound :: Word8) >= numConstructors = ''Word8
108 | fromIntegral (maxBound :: Word16) >= numConstructors = ''Word16
109 | fromIntegral (maxBound :: Word32) >= numConstructors = ''Word32
110 | fromIntegral (maxBound :: Word64) >= numConstructors = ''Word64
111 | otherwise = error "Too many constructors"
113--------------------------------------------------------------------------------
115--------------------------------------------------------------------------------
118 SimpleDataCon Name [Field]
121simplifyDataCon :: DataCon -> SimpleDataCon
122simplifyDataCon (DataCon cname _ _ fields) = SimpleDataCon cname fields
125 = UnitType Name -- 1 constructor and 1 field
126 | TheType SimpleDataCon -- 1 constructor and 1+ fields
127 | MultiType [SimpleDataCon] -- 1+ constructors
130typeOfType :: Type -> [DataCon] -> TypeOfType
131typeOfType headTy [] =
133 ("Attempting to get size with no constructors (" ++
134 (pprint headTy) ++ ")")
135typeOfType _ [DataCon cname _ _ []] = UnitType cname
136typeOfType _ [con@(DataCon _ _ _ _)] = TheType $ simplifyDataCon con
137typeOfType _ cons = MultiType $ map simplifyDataCon cons
139isUnitType :: [DataCon] -> Bool
140isUnitType [DataCon _ _ _ []] = True
143--------------------------------------------------------------------------------
145--------------------------------------------------------------------------------
148int_w8 :: Int -> Word8
152w8_w16 :: Word8 -> Word16
155w8_w32 :: Word8 -> Word32
158w8_w64 :: Word8 -> Word64
162w32_int :: Word32 -> Int
163w32_int = fromIntegral
167c2w = fromIntegral . ord
169shiftAdd :: Bits a => (b -> a) -> [b] -> a
171 foldl' (.|.) zeroBits $
172 map (\(j, x) -> shiftL x (j * 8)) $ zip [0 ..] $ map conv xs
174-- Note: This only works in little endian machines
175xorCmp :: [Word8] -> Name -> Name -> Q Exp
178 x | x < 2 -> [|$(go8 0) == zeroBits|]
179 x | x < 4 -> [|$(go16 0) == zeroBits|]
180 x | x < 8 -> [|$(go32 0) == zeroBits|]
181 _ -> [|$(go64 0) == zeroBits|]
184 go8 i | i >= tagLen = [|zeroBits|]
186 let wIntegral = litIntegral i
187 [|xor (unsafeInlineIO
189 ($(varE off) + $(litIntegral i))
191 ($(wIntegral) :: Word8) .|.
194 | i >= tagLen = [|zeroBits|]
196 | tagLen - i < 2 = go16 (tagLen - 2)
200 (shiftAdd w8_w16 [tag !! i, tag !! (i + 1)] :: Word16)
201 [|xor (unsafeInlineIO
203 ($(varE off) + $(litIntegral i))
205 ($(wIntegral) :: Word16) .|.
208 | i >= tagLen = [|zeroBits|]
210 | tagLen - i < 4 = go32 (tagLen - 4)
221 [|xor (unsafeInlineIO
223 ($(varE off) + $(litIntegral i))
225 ($(wIntegral) :: Word32) .|.
228 | i >= tagLen = [|zeroBits|]
230 | tagLen - i < 8 = go64 (tagLen - 8)
245 [|xor (unsafeInlineIO
247 ($(varE off) + $(litIntegral i))
249 ($(wIntegral) :: Word64) .|.
252--------------------------------------------------------------------------------
253-- Primitive serialization
254--------------------------------------------------------------------------------
256-- XXX Batch serialize? See xorCmp
257serializeW8List :: Name -> Name -> [Word8] -> Q Exp
258serializeW8List off arr w8List = do
259 [|let $(varP (makeN 0)) = $(varE off)
260 in $(doE (fmap makeBind [0 .. (lenW8List - 1)] ++
261 [noBindS ([|pure $(varE (makeN lenW8List))|])]))|]
265 lenW8List = length w8List
268 (varP (makeN (i + 1)))
272 ($(litIntegral (w8List !! i)) :: Word8)|]
274--------------------------------------------------------------------------------
276--------------------------------------------------------------------------------
278litIntegral :: Integral a => a -> Q Exp
279litIntegral = litE . IntegerL . fromIntegral
281getNameBaseLen :: Name -> Word8
282getNameBaseLen cname =
283 let x = length (nameBase cname)
285 then error "Max Constructor Len: 63 characters"
288conEncLen :: Name -> Word8
289conEncLen cname = getNameBaseLen cname + 1
291--------------------------------------------------------------------------------
293--------------------------------------------------------------------------------
295mkSizeOfExpr :: Bool -> TypeOfType -> Q Exp
296mkSizeOfExpr True tyOfTy =
301 [|$(varE _acc) + $(litIntegral (conEncLen cname))|]
305 (caseE (varE _x) [matchCons (varE _acc) con])
306 MultiType constructors -> sizeOfHeadDt constructors
310 sizeOfFields acc fields =
311 foldl' exprGetSize acc $ zip [0..] fields
313 matchCons acc (SimpleDataCon cname fields) =
314 let a = litIntegral (conEncLen cname)
315 b = sizeOfFields acc (map snd fields)
316 expr = [|$(a) + $(b)|]
317 in matchConstructor cname (length fields) expr
320 let acc = [|$(varE _acc)|]
323 (caseE (varE _x) (fmap (matchCons acc) cons))
325mkSizeOfExpr False tyOfTy =
327 UnitType _ -> lamE [varP _acc, wildP] [|$(varE _acc) + 1|]
331 (caseE (varE _x) [matchCons (varE _acc) con])
332 MultiType constructors -> sizeOfHeadDt constructors
336 tagSizeExp numConstructors =
337 litE (IntegerL (fromIntegral (getTagSize numConstructors)))
339 -- XXX fields of the same type can be folded together, will reduce the code
340 -- size when there are many fields of the same type.
341 -- XXX const size fields can be calculated statically.
342 -- XXX This can result in large compilation times due to nesting when there
343 -- are many constructors. We can create a list and sum the list at run time
344 -- to avoid that depending on the number of constructors. Or using a let
345 -- statement for each case may help?
346 -- appE (varE 'sum) (listE (acc : map (exprGetSize (litE (IntegerL 0))) (zip [0..] fields)))
347 sizeOfFields acc fields =
348 foldl' exprGetSize acc $ zip [0..] fields
350 matchCons acc (SimpleDataCon cname fields) =
351 let expr = sizeOfFields acc (map snd fields)
352 in matchConstructor cname (length fields) expr
354 -- XXX We fix VarSize for simplicity. Should be changed later.
356 let numCons = length cons
357 acc = [|$(varE _acc) + $(tagSizeExp numCons)|]
360 (caseE (varE _x) (fmap (matchCons acc) cons))
362--------------------------------------------------------------------------------
364--------------------------------------------------------------------------------
366mkDeserializeExprOne :: SimpleDataCon -> Q Exp
367mkDeserializeExprOne (SimpleDataCon cname fields) =
369 -- Only tag is serialized for unit fields, no actual value
370 [] -> [|pure ($(varE (mkName "i0")), $(conE cname))|]
374 [ fmap makeBind [0 .. (numFields - 1)]
379 [ varE (makeI numFields)
383 [0 .. (numFields - 1)]))
388 numFields = length fields
391 (tupP [varP (makeI (i + 1)), varP (makeA i)])
392 [|deserialize $(varE (makeI i)) $(varE _arr) $(varE _endOffset)|]
395mkDeserializeExpr :: Bool -> Type -> TypeOfType -> Q Exp
396mkDeserializeExpr True headTy tyOfTy =
398 UnitType cname -> deserializeConsExpr [SimpleDataCon cname []]
399 TheType con -> deserializeConsExpr [con]
400 MultiType cons -> deserializeConsExpr cons
404 deserializeConsExpr cons = do
405 conLen <- newName "conLen"
406 off1 <- newName "off1"
407 [|do ($(varP off1), $(varP conLen) :: Word8) <-
409 $(varE _initialOffset)
412 $(multiIfE (map (guardCon conLen off1) cons ++ [catchAll]))|]
418 ("Found invalid tag while peeking (" ++
419 $(lift (pprint headTy)) ++ ")")|]
421 guardCon conLen off con@(SimpleDataCon cname _) = do
422 let lenCname = getNameBaseLen cname
423 tag = map c2w (nameBase cname)
425 [|($(litIntegral lenCname) == $(varE conLen))
426 && $(xorCmp tag off _arr)|]
427 [|let $(varP (makeI 0)) = $(varE off) + $(litIntegral lenCname)
428 in $(mkDeserializeExprOne con)|]
430mkDeserializeExpr False headTy tyOfTy =
434 [|pure ($(varE _initialOffset) + 1, $(conE cname))|]
438 [valD (varP (mkName "i0")) (normalB (varE _initialOffset)) []]
439 (mkDeserializeExprOne con)
442 let lenCons = length cons
443 tagType = getTagType lenCons
446 (tupP [varP (mkName "i0"), varP _tag])
447 [|deserialize $(varE _initialOffset) $(varE _arr) $(varE _endOffset)|]
450 (sigE (varE _tag) (conT tagType))
451 (map peekMatch (zip [0 ..] cons) ++ [peekErr]))
455 match (litP (IntegerL i)) (normalB (mkDeserializeExprOne con)) []
462 ("Found invalid tag while peeking (" ++
463 $(lift (pprint headTy)) ++ ")")|])
466--------------------------------------------------------------------------------
468--------------------------------------------------------------------------------
470mkSerializeExprTag :: Name -> Int -> Q Exp
471mkSerializeExprTag tagType tagVal =
473 $(varE _initialOffset)
475 $((sigE (litE (IntegerL (fromIntegral tagVal))) (conT tagType)))|]
477mkSerializeExprFields :: [Field] -> Q Exp
478mkSerializeExprFields fields =
480 -- Unit constructor, do nothing just tag is enough
481 [] -> [|pure ($(varE (mkName "i0")))|]
484 (fmap makeBind [0 .. (numFields - 1)] ++
485 [noBindS ([|pure $(varE (makeI numFields))|])])
487 numFields = length fields
490 (varP (makeI (i + 1)))
491 [|serialize $(varE (makeI i)) $(varE _arr) $(varE (mkFieldName i))|]
493mkSerializeExpr :: Bool -> TypeOfType -> Q Exp
494mkSerializeExpr True tyOfTy =
500 [serializeDataCon (SimpleDataCon cname [])]
505 [serializeDataCon con]
510 (map serializeDataCon cons)
514 serializeDataCon (SimpleDataCon cname fields) = do
515 let tagLen8 = getNameBaseLen cname
516 conEnc = tagLen8 : map c2w (nameBase cname)
522 (serializeW8List _initialOffset _arr conEnc)
523 , noBindS (mkSerializeExprFields fields)
526mkSerializeExpr False tyOfTy =
529 UnitType _ -> [|pure ($(varE _initialOffset) + 1)|]
531 (TheType (SimpleDataCon cname fields)) ->
533 [valD (varP (mkName "i0")) (normalB (varE _initialOffset)) []]
539 (mkSerializeExprFields fields)
542 (MultiType cons) -> do
543 let lenCons = length cons
544 tagType = getTagType lenCons
547 (map (\(tagVal, (SimpleDataCon cname fields)) ->
553 (mkSerializeExprTag tagType tagVal)
554 , noBindS (mkSerializeExprFields fields)
558--------------------------------------------------------------------------------
560--------------------------------------------------------------------------------
562-- | A general function to derive Serialize instances where you can control
563-- which Constructors of the datatype to consider and what the Context for the
564-- 'Serialize' instance would be.
566-- Consider the datatype:
568-- data CustomDataType a b
570-- | CDTConstructor2 Bool
571-- | CDTConstructor3 Bool b
572-- deriving (Show, Eq)
577-- $(deriveSerializeInternal
579-- [AppT (ConT ''Serialize) (VarT (mkName "b"))]
581-- (AppT (ConT ''CustomDataType) (VarT (mkName "a")))
582-- (VarT (mkName "b")))
583-- [ DataCon 'CDTConstructor1 [] [] []
584-- , DataCon 'CDTConstructor2 [] [] [(Nothing, (ConT ''Bool))]
589-- [(Nothing, (ConT ''Bool)), (Nothing, (VarT (mkName "b")))]
592deriveSerializeInternal ::
593 Config -> Cxt -> Type -> [DataCon] -> Q [Dec]
594deriveSerializeInternal (Config{..}) preds headTy cons = do
595 sizeOfMethod <- mkSizeOfExpr constructorTagAsString (typeOfType headTy cons)
597 mkDeserializeExpr constructorTagAsString headTy (typeOfType headTy cons)
599 mkSerializeExpr constructorTagAsString (typeOfType headTy cons)
601 -- INLINE on sizeOf actually worsens some benchmarks, and improves
603 [ PragmaD (InlineP 'size inlineSize FunLike AllPhases)
604 , FunD 'size [Clause [] (NormalB sizeOfMethod) []]
605 , PragmaD (InlineP 'deserialize inlineDeserialize FunLike AllPhases)
609 (if isUnitType cons && not constructorTagAsString
610 then [VarP _initialOffset, WildP, WildP]
611 else [VarP _initialOffset, VarP _arr, VarP _endOffset])
615 , PragmaD (InlineP 'serialize inlineSerialize FunLike AllPhases)
619 (if isUnitType cons && not constructorTagAsString
620 then [VarP _initialOffset, WildP, WildP]
621 else [VarP _initialOffset, VarP _arr, VarP _val])
626 return [plainInstanceD preds (AppT (ConT ''Serialize) headTy) methods]
628-- | Config to control how the 'Serialize' instance is generated.
631 { unconstrained :: [String]
632 -- ^ Type variables that should not have the 'Serialize' constraint.
635 -- data CustomDataType a b c d = CustomDataType a c
639 -- conf = defaultConf {unconstrained = ["b", "d"]}
640 -- \$(deriveSerializeWith conf ''CustomDataType)
644 -- instance (Serialize a, Serialize c) => Serialize (CustomDataType a b c d) where
647 , specializations :: [(String, Type)]
648 -- ^ Specialize the type variable with the given type. All type
649 -- variables listed here will not have the 'Serialize' constriant as
650 -- they are specialized.
653 -- data CustomDataType f a = CustomDataType (f a)
657 -- conf = defaultConf {specializations = [("f", ''Identity)]}
658 -- \$(deriveSerializeWith conf ''CustomDataType)
662 -- instance (Serialize a) => Serialize (CustomDataType Identity a) where
666 -- @f@ is replaced with 'Identity' and becomes unconstrained.
667 , inlineSize :: Inline
668 -- ^ Inline value for 'size'. Default is Inline.
669 , inlineSerialize :: Inline
670 -- ^ Inline value for 'serialize'. Default is Inline.
671 , inlineDeserialize :: Inline
672 -- ^ Inline value for 'deserialize'. Default is Inline.
673 , constructorTagAsString :: Bool
674 -- ^ __Experimental__
676 -- If True, encode constructors using the constructor names as Latin-1
678 , recordSyntaxWithHeader :: Bool
679 -- ^ __Experimental__
681 -- If True, encode the keys of the record as a header and then
682 -- serialize the data.
685defaultConfig :: Config
689 , specializations = []
690 , inlineSize = Inline
691 , inlineSerialize = Inline
692 , inlineDeserialize = Inline
693 , constructorTagAsString = False
694 , recordSyntaxWithHeader = False
697-- | Similar to 'deriveSerialize,' but take a 'Config' to control how
698-- the instance is generated.
700-- Usage: @$(deriveSerializeWith config ''CustomDataType)@
701deriveSerializeWith :: Config -> Name -> Q [Dec]
702deriveSerializeWith conf@(Config {..}) name = do
703 dt <- reifyDataType name
704 let preds = map (unboxPred . VarT) (filterOutVars (dtTvs dt))
705 headTy = appsT (ConT name) (map substituteVar (dtTvs dt))
707 deriveSerializeInternal conf preds headTy cons
710 allUnconstrainedTypeVars =
711 unconstrained ++ map fst specializations
714 $ filter (not . flip elem allUnconstrainedTypeVars)
717 case lookup (nameBase v) specializations of
722#if MIN_VERSION_template_haskell(2,10,0)
723 AppT (ConT ''Serialize) ty
725 ClassP ''Serialize [ty]
728-- | Template haskell helper to create instances of 'Serialize' automatically.
730-- Consider the datatype:
732-- data CustomDataType a b c = ...
735-- Usage: @$(deriveSerialize ''CustomDataType)@
737-- Note: All type variables automatcally get an "Serialize" constraint.
738-- The derived code will look like the following,
740-- instance (Serialize a, Serialize b, Serialize c) => Serialize (CustomDataType a b c) where
744-- To control which type variables don't get the Serialize constraint, use
745-- 'deriveSerializeWith'.
747-- >>> import qualified Streamly.Internal.Data.Serialize.TH as Serialize
748-- >>> deriveSerialize = Serialize.deriveSerializeWith Serialize.defaultConfig
749deriveSerialize :: Name -> Q [Dec]
750deriveSerialize name = deriveSerializeWith defaultConfig name
keysesc to close
- j k
- move down, up
- enter
- open
- esc
- back, or up a level
- 1 2 3 4
- readme, files, commits, refs
- /
- filter
- y
- copy the clone command
- c
- compare the chosen commit with another (or shift-click both)
- ?
- show or hide this list