Add option to reject unknown fields (#207) (#749)

This commit adds rejectUnknownFields as a Option that affects parsing.
Parsing will succeed only if all of the fields in the raw JSON value
are present in the record to which it will be parsed.

It supports both Generics and TH, though the TH implementation has less
detailed messaging. This seems to be a pattern througout though, so it
should be consistent with other parsing errors.
This commit is contained in:
rmanne 2020-02-06 09:00:31 -08:00 committed by GitHub
parent 4945c67152
commit 817ad94716
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 240 additions and 26 deletions

View File

@ -143,7 +143,7 @@ import Language.Haskell.TH.Syntax (mkNameG_tc)
import Text.Printf (printf)
import qualified Data.Aeson.Encoding.Internal as E
import qualified Data.Foldable as F (all)
import qualified Data.HashMap.Strict as H (lookup, toList)
import qualified Data.HashMap.Strict as H (difference, fromList, keys, lookup, toList)
import qualified Data.List.NonEmpty as NE (length, reverse)
import qualified Data.Map as M (fromList, keys, lookup , singleton, size)
import qualified Data.Semigroup as Semigroup (Option(..))
@ -922,12 +922,32 @@ parseRecord :: JSONClass
-> Name
-> [Name]
-> Name
-> Bool
-> ExpQ
parseRecord jc tvMap argTys opts tName conName fields obj =
parseRecord jc tvMap argTys opts tName conName fields obj inTaggedObject =
(if rejectUnknownFields opts
then infixApp checkUnknownRecords [|(>>)|]
else id) $
foldl' (\a b -> infixApp a [|(<*>)|] b)
(infixApp (conE conName) [|(<$>)|] x)
xs
where
tagFieldNameAppender =
if inTaggedObject then (tagFieldName (sumEncoding opts) :) else id
knownFields = appE [|H.fromList|] $ listE $
map (\knownName -> tupE [appE [|T.pack|] $ litE $ stringL knownName, [|()|]]) $
tagFieldNameAppender $ map nameBase fields
checkUnknownRecords =
caseE (appE [|H.keys|] $ infixApp (varE obj) [|H.difference|] knownFields)
[ match (listP []) (normalB [|return ()|]) []
, newName "unknownFields" >>=
\unknownFields -> match (varP unknownFields)
(normalB $ appE [|fail|] $ infixApp
(litE (stringL "Unknown fields: "))
[|(++)|]
(appE [|show|] (varE unknownFields)))
[]
]
x:xs = [ [|lookupField|]
`appE` dispatchParseJSON jc conName tvMap argTy
`appE` litE (stringL $ show tName)
@ -1002,7 +1022,7 @@ parseArgs jc tvMap tName opts
, constructorFields = argTys }
(Left (_, obj)) = do
argTys' <- mapM resolveTypeSynonyms argTys
parseRecord jc tvMap argTys' opts tName conName fields obj
parseRecord jc tvMap argTys' opts tName conName fields obj True
parseArgs jc tvMap tName opts
info@ConstructorInfo { constructorName = conName
, constructorVariant = RecordConstructor fields
@ -1017,7 +1037,7 @@ parseArgs jc tvMap tName opts
argTys' <- mapM resolveTypeSynonyms argTys
caseE (varE valName)
[ match (conP 'Object [varP obj]) (normalB $
parseRecord jc tvMap argTys' opts tName conName fields obj) []
parseRecord jc tvMap argTys' opts tName conName fields obj False) []
, matchFailed tName conName "Object"
]

View File

@ -124,6 +124,7 @@ module Data.Aeson.Types
, sumEncoding
, unwrapUnaryRecords
, tagSingleConstructors
, rejectUnknownFields
-- ** Options utilities
, SumEncoding(..)

View File

@ -11,6 +11,7 @@
{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeOperators #-}
{-# LANGUAGE TupleSections #-}
{-# LANGUAGE UndecidableInstances #-}
{-# LANGUAGE ViewPatterns #-}
@ -1200,9 +1201,9 @@ class FromTaggedObject' arity f isRecord where
:: String :* ConName :* TypeName :* Options :* FromArgs arity a
-> Object -> Tagged isRecord (Parser (f a))
instance (RecordFromJSON arity f) => FromTaggedObject' arity f True where
instance (RecordFromJSON arity f, FieldNames f) => FromTaggedObject' arity f True where
-- Records are unpacked in the tagged object
parseFromTaggedObject' (_ :* p) = Tagged . recordParseJSON p
parseFromTaggedObject' (_ :* p) = Tagged . recordParseJSON (True :* p)
instance (ConsFromJSON arity f) => FromTaggedObject' arity f False where
-- Nonnullary nonrecords are encoded in the contents field
@ -1243,11 +1244,11 @@ instance OVERLAPPING_
) => ConsFromJSON' arity (S1 s a) True where
consParseJSON' p@(cname :* tname :* opts :* fargs)
| unwrapUnaryRecords opts = Tagged . fmap M1 . gParseJSON opts fargs
| otherwise = Tagged . withObject (showCons cname tname) (recordParseJSON p)
| otherwise = Tagged . withObject (showCons cname tname) (recordParseJSON (False :* p))
instance RecordFromJSON arity f => ConsFromJSON' arity f True where
consParseJSON' p@(cname :* tname :* _) =
Tagged . withObject (showCons cname tname) (recordParseJSON p)
Tagged . withObject (showCons cname tname) (recordParseJSON (False :* p))
instance OVERLAPPING_
ConsFromJSON' arity U1 False where
@ -1273,21 +1274,55 @@ instance (ProductFromJSON arity f, ProductSize f
--------------------------------------------------------------------------------
class FieldNames f where
fieldNames :: f a -> [Text] -> [Text]
instance (FieldNames a, FieldNames b) => FieldNames (a :*: b) where
fieldNames _ =
fieldNames (undefined :: a x) .
fieldNames (undefined :: b y)
instance (Selector s) => FieldNames (S1 s f) where
fieldNames _ = (pack (selName (undefined :: M1 _i s _f _p)) :)
class RecordFromJSON arity f where
recordParseJSON
:: Bool :* ConName :* TypeName :* Options :* FromArgs arity a
-> Object -> Parser (f a)
instance ( FieldNames f
, RecordFromJSON' arity f
) => RecordFromJSON arity f where
recordParseJSON (fromTaggedSum :* p@(cname :* tname :* opts :* _)) =
\obj -> checkUnknown obj >> recordParseJSON' p obj
where
knownFields :: H.HashMap Text ()
knownFields = H.fromList $ map (,()) $
fieldNames (undefined :: f a)
[pack (tagFieldName (sumEncoding opts)) | fromTaggedSum]
checkUnknown =
if not (rejectUnknownFields opts)
then \_ -> return ()
else \obj -> case H.keys (H.difference obj knownFields) of
[] -> return ()
unknownFields -> contextCons cname tname $
fail ("unknown fields: " ++ show unknownFields)
class RecordFromJSON' arity f where
recordParseJSON'
:: ConName :* TypeName :* Options :* FromArgs arity a
-> Object -> Parser (f a)
instance ( RecordFromJSON arity a
, RecordFromJSON arity b
) => RecordFromJSON arity (a :*: b) where
recordParseJSON p obj =
(:*:) <$> recordParseJSON p obj
<*> recordParseJSON p obj
instance ( RecordFromJSON' arity a
, RecordFromJSON' arity b
) => RecordFromJSON' arity (a :*: b) where
recordParseJSON' p obj =
(:*:) <$> recordParseJSON' p obj
<*> recordParseJSON' p obj
instance OVERLAPPABLE_ (Selector s, GFromJSON arity a) =>
RecordFromJSON arity (S1 s a) where
recordParseJSON (cname :* tname :* opts :* fargs) obj = do
RecordFromJSON' arity (S1 s a) where
recordParseJSON' (cname :* tname :* opts :* fargs) obj = do
fv <- contextCons cname tname (obj .: label)
M1 <$> gParseJSON opts fargs fv <?> Key label
where
@ -1295,16 +1330,16 @@ instance OVERLAPPABLE_ (Selector s, GFromJSON arity a) =>
sname = selName (undefined :: M1 _i s _f _p)
instance INCOHERENT_ (Selector s, FromJSON a) =>
RecordFromJSON arity (S1 s (K1 i (Maybe a))) where
recordParseJSON (_ :* _ :* opts :* _) obj = M1 . K1 <$> obj .:? pack label
RecordFromJSON' arity (S1 s (K1 i (Maybe a))) where
recordParseJSON' (_ :* _ :* opts :* _) obj = M1 . K1 <$> obj .:? pack label
where
label = fieldLabelModifier opts sname
sname = selName (undefined :: M1 _i s _f _p)
-- Parse an Option like a Maybe.
instance INCOHERENT_ (Selector s, FromJSON a) =>
RecordFromJSON arity (S1 s (K1 i (Semigroup.Option a))) where
recordParseJSON p obj = wrap <$> recordParseJSON p obj
RecordFromJSON' arity (S1 s (K1 i (Semigroup.Option a))) where
recordParseJSON' p obj = wrap <$> recordParseJSON' p obj
where
wrap :: S1 s (K1 i (Maybe a)) p -> S1 s (K1 i (Semigroup.Option a)) p
wrap (M1 (K1 a)) = M1 (K1 (Semigroup.Option a))

View File

@ -64,6 +64,7 @@ module Data.Aeson.Types.Internal
, sumEncoding
, unwrapUnaryRecords
, tagSingleConstructors
, rejectUnknownFields
)
, SumEncoding(..)
@ -624,10 +625,14 @@ data Options = Options
, tagSingleConstructors :: Bool
-- ^ Encode types with a single constructor as sums,
-- so that `allNullaryToStringTag` and `sumEncoding` apply.
, rejectUnknownFields :: Bool
-- ^ Applies only to 'Data.Aeson.FromJSON' instances. If a field appears in
-- the parsed object map, but does not appear in the target object, parsing
-- will fail, with an error message indicating which fields were unknown.
}
instance Show Options where
show (Options f c a o s u t) =
show (Options f c a o s u t r) =
"Options {"
++ intercalate ", "
[ "fieldLabelModifier =~ " ++ show (f "exampleField")
@ -637,6 +642,7 @@ instance Show Options where
, "sumEncoding = " ++ show s
, "unwrapUnaryRecords = " ++ show u
, "tagSingleConstructors = " ++ show t
, "rejectUnknownFields = " ++ show r
]
++ "}"
@ -718,6 +724,7 @@ data JSONKeyOptions = JSONKeyOptions
-- , 'sumEncoding' = 'defaultTaggedObject'
-- , 'unwrapUnaryRecords' = False
-- , 'tagSingleConstructors' = False
-- , 'rejectUnknownFields' = False
-- }
-- @
defaultOptions :: Options
@ -729,6 +736,7 @@ defaultOptions = Options
, sumEncoding = defaultTaggedObject
, unwrapUnaryRecords = False
, tagSingleConstructors = False
, rejectUnknownFields = False
}
-- | Default 'TaggedObject' 'SumEncoding' options:

View File

@ -243,6 +243,31 @@ gSomeTypeToEncodingOmitNothingFields :: SomeType Int -> Encoding
gSomeTypeToEncodingOmitNothingFields = genericToEncoding optsOmitNothingFields
thSomeTypeParseJSONRejectUnknownFields :: Value -> Parser (SomeType Int)
thSomeTypeParseJSONRejectUnknownFields = $(mkParseJSON optsRejectUnknownFields ''SomeType)
gSomeTypeParseJSONRejectUnknownFields :: Value -> Parser (SomeType Int)
gSomeTypeParseJSONRejectUnknownFields = genericParseJSON optsRejectUnknownFields
--------------------------------------------------------------------------------
-- Foo decoders
--------------------------------------------------------------------------------
thFooParseJSONRejectUnknownFields :: Value -> Parser Foo
thFooParseJSONRejectUnknownFields = $(mkParseJSON optsRejectUnknownFields ''Foo)
gFooParseJSONRejectUnknownFields :: Value -> Parser Foo
gFooParseJSONRejectUnknownFields = genericParseJSON optsRejectUnknownFields
thFooParseJSONRejectUnknownFieldsTagged :: Value -> Parser Foo
thFooParseJSONRejectUnknownFieldsTagged = $(mkParseJSON optsRejectUnknownFieldsTagged ''Foo)
gFooParseJSONRejectUnknownFieldsTagged :: Value -> Parser Foo
gFooParseJSONRejectUnknownFieldsTagged = genericParseJSON optsRejectUnknownFieldsTagged
--------------------------------------------------------------------------------
-- Option fields
--------------------------------------------------------------------------------

View File

@ -136,6 +136,29 @@ outputGeneric choice = concat
, "[1,"
]
, testWithSomeType "SomeType (reject unknown fields)"
(select
thSomeTypeParseJSONRejectUnknownFields
gSomeTypeParseJSONRejectUnknownFields)
[ "{\"tag\": \"record\", \"testOne\": 1.0, \"testZero\": 1}"
, "{\"testZero\": 1}"
, "{\"tag\": \"record\", \"testone\": true, \"testtwo\": null, \"testthree\": null}"
]
, testWithFoo "Foo (reject unknown fields)"
(select
thFooParseJSONRejectUnknownFields
gFooParseJSONRejectUnknownFields)
[ "{\"tag\": \"foo\"}"
]
, testWithFoo "Foo (reject unknown fields, tagged single)"
(select
thFooParseJSONRejectUnknownFieldsTagged
gFooParseJSONRejectUnknownFieldsTagged)
[ "{\"tag\": \"foo\", \"unknownField\": 0}"
]
, testWith "EitherTextInt"
(select
thEitherTextIntParseJSONUntaggedValue
@ -196,3 +219,6 @@ testFor name _ = testWith name (parseJSON :: Value -> Parser a)
testWithSomeType :: String -> (Value -> Parser (SomeType Int)) -> [L.ByteString] -> Output
testWithSomeType = testWith
testWithFoo :: String -> (Value -> Parser Foo) -> [L.ByteString] -> Output
testWithFoo = testWith

View File

@ -29,7 +29,7 @@ import Data.Hashable.Time ()
-- "System" types.
instance Arbitrary DotNetTime where
arbitrary = DotNetTime `liftM` arbitrary
arbitrary = DotNetTime `fmap` arbitrary
shrink = map DotNetTime . shrink . fromDotNetTime
-- | Compare timezone part only on 'timeZoneMinutes'

View File

@ -55,3 +55,14 @@ optsOptionField = optsDefault
{ fieldLabelModifier = const "field"
, omitNothingFields = True
}
optsRejectUnknownFields :: Options
optsRejectUnknownFields = optsDefault
{ rejectUnknownFields = True
}
optsRejectUnknownFieldsTagged :: Options
optsRejectUnknownFieldsTagged = optsDefault
{ rejectUnknownFields = True
, tagSingleConstructors = True
}

View File

@ -33,15 +33,15 @@ import Data.Aeson.Parser
( json, jsonLast, jsonAccum, jsonNoDup
, json', jsonLast', jsonAccum', jsonNoDup')
import Data.Aeson.Types
( Options(..), Result(Success), ToJSON(..), Value(Array, Bool, Null, Object)
, camelTo, camelTo2, defaultOptions, formatPath, formatRelativePath
, omitNothingFields, parse)
( Options(..), Result(Success, Error), ToJSON(..)
, Value(Array, Bool, Null, Number, Object, String), camelTo, camelTo2
, defaultOptions, formatPath, formatRelativePath, omitNothingFields, parse)
import Data.Attoparsec.ByteString (Parser, parseOnly)
import Data.Char (toUpper)
import Data.Either.Compat (isLeft, isRight)
import Data.Hashable (hash)
import Data.HashMap.Strict (HashMap)
import Data.List (sort)
import Data.List (sort, isSuffixOf)
import Data.Maybe (fromMaybe)
import Data.Scientific (Scientific, scientific)
import Data.Tagged (Tagged(..))
@ -112,6 +112,7 @@ tests = testGroup "unit" [
, testGroup "SingleMaybeField" singleMaybeField
, testCase "withEmbeddedJSON" withEmbeddedJSONTest
, testCase "SingleFieldCon" singleFieldCon
, testGroup "UnknownFields" unknownFields
, testGroup "Ordering of object keys" keyOrdering
, testCase "Ratio with denominator 0" ratioDenominator0
, testCase "Rational parses number" rationalNumber
@ -539,6 +540,7 @@ showOptions =
++ ", sumEncoding = TaggedObject {tagFieldName = \"tag\", contentsFieldName = \"contents\"}"
++ ", unwrapUnaryRecords = False"
++ ", tagSingleConstructors = False"
++ ", rejectUnknownFields = False"
++ "}")
(show defaultOptions)
@ -583,6 +585,79 @@ singleFieldCon :: Assertion
singleFieldCon =
assertEqual "fromJSON" (Right (SingleFieldCon 0)) (eitherDecode "0")
newtype UnknownFields = UnknownFields { knownField :: Int }
deriving (Eq, Show, Generic)
newtype UnknownFieldsTag = UnknownFieldsTag { tag :: Int }
deriving (Eq, Show, Generic)
newtype UnknownFieldsUnaryTagged = UnknownFieldsUnaryTagged { knownFieldUnaryTagged :: Int }
deriving (Eq, Show, Generic)
data UnknownFieldsSum
= UnknownFields1 { knownField1 :: Int }
| UnknownFields2 { knownField2 :: Int }
deriving (Eq, Show, Generic)
unknownFields :: [TestTree]
unknownFields = concat
[ testsUnary
"unary-unknown"
(object [("knownField", Number 1), ("unknownField", Number 1)])
(Error "nknown fields: [\"unknownField\"]" :: Result UnknownFields)
, testsUnary
"unary-unknown-tag"
(object [("knownField", Number 1), ("tag", String "UnknownFields")])
(Error "nknown fields: [\"tag\"]" :: Result UnknownFields)
, testsUnaryTag
"unary-explicit-tag"
(object [("tag", Number 1)])
(Success $ UnknownFieldsTag 1)
, testsSum
"sum-tag"
(object [("knownField1", Number 1), ("tag", String "UnknownFields1")])
(Success $ UnknownFields1 1)
, testsSum
"sum-unknown-in-branch"
(object [("knownField1", Number 1), ("knownField2", Number 1), ("tag", String "UnknownFields1")])
(Error "nknown fields: [\"knownField2\"]" :: Result UnknownFieldsSum)
, testsSum
"sum-unknown"
(object [("knownField1", Number 1), ("unknownField", Number 1), ("tag", String "UnknownFields1")])
(Error "nknown fields: [\"unknownField\"]" :: Result UnknownFieldsSum)
, testsTagged
"unary-tagged"
(object [("knownFieldUnaryTagged", Number 1), ("tag", String "UnknownFieldsUnaryTagged")])
(Success $ UnknownFieldsUnaryTagged 1)
, -- Just a case to verify that the tag isn't optional, this is likely already tested by other unit tests
testsTagged
"unary-tagged-notag"
(object [("knownFieldUnaryTagged", Number 1)])
(Error "key \"tag\" not found" :: Result UnknownFieldsUnaryTagged)
, testsTagged
"unary-tagged-unknown"
(object [ ("knownFieldUnaryTagged", Number 1), ("unknownField", Number 1)
, ("tag", String "UnknownFieldsUnaryTagged")])
(Error "nknown fields: [\"unknownField\"]" :: Result UnknownFieldsUnaryTagged)
]
where
opts = defaultOptions{rejectUnknownFields=True}
taggedOpts = opts{tagSingleConstructors=True}
assertApprox :: (Show a, Eq a) => Result a -> Result a -> IO ()
assertApprox (Error expected) (Error actual) | expected `isSuffixOf` actual = return ()
assertApprox expected actual = assertEqual "fromJSON" expected actual
testsBase :: (Show a, Eq a) => (Value -> Result a) -> (Value -> Result a)
-> String -> Value -> Result a -> [TestTree]
testsBase th g name value expected =
[ testCase (name ++ "-th") $ assertApprox expected (th value)
, testCase (name ++ "-generic") $ assertApprox expected (g value)
]
testsUnary :: String -> Value -> Result UnknownFields -> [TestTree]
testsUnary = testsBase fromJSON (parse (genericParseJSON opts))
testsUnaryTag :: String -> Value -> Result UnknownFieldsTag -> [TestTree]
testsUnaryTag = testsBase fromJSON (parse (genericParseJSON opts))
testsSum :: String -> Value -> Result UnknownFieldsSum -> [TestTree]
testsSum = testsBase fromJSON (parse (genericParseJSON opts))
testsTagged :: String -> Value -> Result UnknownFieldsUnaryTagged -> [TestTree]
testsTagged = testsBase fromJSON (parse (genericParseJSON taggedOpts))
testParser :: (Eq a, Show a)
=> String -> Parser a -> S.ByteString -> Either String a -> TestTree
testParser name json_ s expected =
@ -678,3 +753,8 @@ deriveToJSON defaultOptions ''Foo
deriveToJSON1 defaultOptions ''Foo
deriveJSON defaultOptions{omitNothingFields=True,unwrapUnaryRecords=True} ''SingleMaybeField
deriveJSON defaultOptions{rejectUnknownFields=True} ''UnknownFields
deriveJSON defaultOptions{rejectUnknownFields=True} ''UnknownFieldsTag
deriveJSON defaultOptions{tagSingleConstructors=True,rejectUnknownFields=True} ''UnknownFieldsUnaryTagged
deriveJSON defaultOptions{rejectUnknownFields=True} ''UnknownFieldsSum

View File

@ -31,6 +31,14 @@ Error in $: parsing Types.SomeType failed, expected a 2-element Array, but encou
Error in $: parsing Types.SomeType failed, expected Array, but encountered Object
Error in $: not enough input. Expecting ',' or ']'
Error in $: not enough input. Expecting json list value
SomeType (reject unknown fields)
Error in $: parsing Types.SomeType(Record) failed, unknown fields: ["testZero"]
Error in $: parsing Types.SomeType failed, expected Object with key "tag" containing one of ["nullary","unary","product","record","list"], key "tag" not found
Error in $: parsing Types.SomeType(Record) failed, unknown fields: ["testtwo","testone","testthree"]
Foo (reject unknown fields)
Error in $: parsing Types.Foo(Foo) failed, unknown fields: ["tag"]
Foo (reject unknown fields, tagged single)
Error in $: parsing Types.Foo(Foo) failed, unknown fields: ["unknownField"]
EitherTextInt
Error in $: parsing Types.EitherTextInt(NoneNullary) failed, expected tag "nonenullary", but found tag "X"
Error in $: parsing Types.EitherTextInt(NoneNullary) failed, expected String, but encountered Array