on master help sign in
Add bounds checks for unsafe conversions from Scientific
commitd35444587698e5961d1f266e3477cf627f7575f5 authorAdam Bergmark <adam@bergmark.nl> dateFri 8 Jun 2018, 16:27 (8 years ago) parent161f14d
5 files changed+102−42
Data/Aeson/Encoding/Builder.hs+1−1 Data/Aeson/Types/FromJSON.hs+47−39 aeson.cabal+1−1 changelog.md+16−1 tests/UnitTests.hs+37
Data/Aeson/Encoding/Builder.hs+1−1
@@ -133,7 +133,7 @@ c2w c = fromIntegral (ord c)
133133-- | Encode a JSON number.
134134scientific :: Scientific -> Builder
135135scientific s
136− | e < 0 = scientificBuilder s
136+ | e < 0 || e > 1024 = scientificBuilder s
137137 | otherwise = B.integerDec (coefficient s * 10 ^ e)
138138 where
139139 e = base10Exponent s
Data/Aeson/Types/FromJSON.hs+47−39
@@ -85,7 +85,6 @@ import Data.Aeson.Internal.Functions (mapKey)
8585import Data.Aeson.Parser.Internal (eitherDecodeWith, jsonEOF)
8686import Data.Aeson.Types.Generic
8787import Data.Aeson.Types.Internal
88−import Data.Attoparsec.Number (Number(..))
8988import Data.Bits (unsafeShiftR)
9089import Data.Fixed (Fixed, HasResolution)
9190import Data.Functor.Compose (Compose(..))
@@ -99,7 +98,7 @@ import Data.Maybe (fromMaybe)
9998import Data.Semigroup ((<>))
10099import Data.Proxy (Proxy(..))
101100import Data.Ratio ((%), Ratio)
102−import Data.Scientific (Scientific)
101+import Data.Scientific (Scientific, base10Exponent)
103102import Data.Tagged (Tagged(..))
104103import Data.Text (Text, pack, unpack)
105104import Data.Time (Day, DiffTime, LocalTime, NominalDiffTime, TimeOfDay, UTCTime, ZonedTime)
@@ -172,15 +171,6 @@ parseIndexedJSONPair keyParser valParser idx value = p value <?> Index idx
172171parseJSONElemAtIndex :: (Value -> Parser a) -> Int -> V.Vector Value -> Parser a
173172parseJSONElemAtIndex p idx ary = p (V.unsafeIndex ary idx) <?> Index idx
174173
175−scientificToNumber :: Scientific -> Number
176−scientificToNumber s
177− | e < 0 = D $ Scientific.toRealFloat s
178− | otherwise = I $ c * 10 ^ e
179− where
180− e = Scientific.base10Exponent s
181− c = Scientific.coefficient s
182−{-# INLINE scientificToNumber #-}
183−
184174parseRealFloat :: RealFloat a => String -> Value -> Parser a
185175parseRealFloat _ (Number s) = pure $ Scientific.toRealFloat s
186176parseRealFloat _ Null = pure (0/0)
@@ -196,7 +186,7 @@ parseIntegralFromScientific expected s =
196186
197187parseIntegral :: Integral a => String -> Value -> Parser a
198188parseIntegral expected =
199− withScientific expected $ parseIntegralFromScientific expected
189+ withBoundedScientific expected $ parseIntegralFromScientific expected
200190{-# INLINE parseIntegral #-}
201191
202192parseBoundedIntegralFromScientific :: (Bounded a, Integral a) => String -> Scientific -> Parser a
@@ -218,8 +208,13 @@ parseScientificText
218208 . T.encodeUtf8
219209
220210parseIntegralText :: Integral a => String -> Text -> Parser a
221−parseIntegralText expected t =
222− parseScientificText t >>= parseIntegralFromScientific expected
211+parseIntegralText expected t
212+ = parseScientificText t
213+ >>= rejectLargeExponent
214+ >>= parseIntegralFromScientific expected
215+ where
216+ rejectLargeExponent :: Scientific -> Parser Scientific
217+ rejectLargeExponent s = withBoundedScientific expected pure (Number s)
223218{-# INLINE parseIntegralText #-}
224219
225220parseBoundedIntegralText :: (Bounded a, Integral a) => String -> Text -> Parser a
@@ -648,11 +643,30 @@ withArray expected _ v = typeMismatch expected v
648643-- | @'withScientific' expected f value@ applies @f@ to the 'Scientific' number
649644-- when @value@ is a 'Number' and fails using @'typeMismatch' expected@
650645-- otherwise.
646+-- .
647+-- /Warning/: If you are converting from a scientific to an unbounded
648+-- type such as 'Integer' you may want to add a restriction on the
649+-- size of the exponent (see 'withBoundedScientific') to prevent
650+-- malicious input from filling up the memory of the target system.
651651withScientific :: String -> (Scientific -> Parser a) -> Value -> Parser a
652652withScientific _ f (Number scientific) = f scientific
653653withScientific expected _ v = typeMismatch expected v
654654{-# INLINE withScientific #-}
655655
656+-- | @'withBoundedScientific' expected f value@ applies @f@ to the 'Scientific' number
657+-- when @value@ is a 'Number' and fails using @'typeMismatch' expected@
658+-- otherwise.
659+--
660+-- The conversion will also fail wyth a @'typeMismatch' if the
661+-- 'Scientific' exponent is larger than 1024.
662+withBoundedScientific :: String -> (Scientific -> Parser a) -> Value -> Parser a
663+withBoundedScientific _ f v@(Number scientific) =
664+ if base10Exponent scientific > 1024
665+ then typeMismatch "a number with exponent <= 1024" v
666+ else f scientific
667+withBoundedScientific expected _ v = typeMismatch expected v
668+{-# INLINE withBoundedScientific #-}
669+
656670-- | @'withBool' expected f value@ applies @f@ to the 'Bool' when @value@ is a
657671-- 'Bool' and fails using @'typeMismatch' expected@ otherwise.
658672withBool :: String -> (Bool -> Parser a) -> Value -> Parser a
@@ -1232,12 +1246,6 @@ instance FromJSONKey Double where
12321246 "-Infinity" -> pure (negate 1/0)
12331247 _ -> Scientific.toRealFloat <$> parseScientificText t
12341248
1235−instance FromJSON Number where
1236− parseJSON (Number s) = pure $ scientificToNumber s
1237− parseJSON Null = pure (D (0/0))
1238− parseJSON v = typeMismatch "Number" v
1239− {-# INLINE parseJSON #-}
1240−
12411249instance FromJSON Float where
12421250 parseJSON = parseRealFloat "Float"
12431251 {-# INLINE parseJSON #-}
@@ -1258,12 +1266,12 @@ instance (FromJSON a, Integral a) => FromJSON (Ratio a) where
12581266 else pure $ numerator % denominator
12591267 {-# INLINE parseJSON #-}
12601268
1261−-- | /WARNING:/ Only parse fixed-precision numbers from trusted input
1262−-- since an attacker could easily fill up the memory of the target
1263−-- system by specifying a scientific number with a big exponent like
1264−-- @1e1000000000@.
1269+-- | This instance includes a bounds check to prevent maliciously
1270+-- large inputs to fill up the memory of the target system. You can
1271+-- newtype 'Scientific' and provide your own instance using
1272+-- 'withScientific' if you want to allow larger inputs.
12651273instance HasResolution a => FromJSON (Fixed a) where
1266− parseJSON = withScientific "Fixed" $ pure . realToFrac
1274+ parseJSON = withBoundedScientific "Fixed" $ pure . realToFrac
12671275 {-# INLINE parseJSON #-}
12681276
12691277instance FromJSON Int where
@@ -1273,10 +1281,10 @@ instance FromJSON Int where
12731281instance FromJSONKey Int where
12741282 fromJSONKey = FromJSONKeyTextParser $ parseBoundedIntegralText "Int"
12751283
1276−-- | /WARNING:/ Only parse Integers from trusted input since an
1277−-- attacker could easily fill up the memory of the target system by
1278−-- specifying a scientific number with a big exponent like
1279−-- @1e1000000000@.
1284+-- | This instance includes a bounds check to prevent maliciously
1285+-- large inputs to fill up the memory of the target system. You can
1286+-- newtype 'Scientific' and provide your own instance using
1287+-- 'withScientific' if you want to allow larger inputs.
12801288instance FromJSON Integer where
12811289 parseJSON = parseIntegral "Integer"
12821290 {-# INLINE parseJSON #-}
@@ -1715,21 +1723,21 @@ instance FromJSONKey UTCTime where
17151723 fromJSONKey = FromJSONKeyTextParser (Time.run Time.utcTime)
17161724
17171725
1718−-- | /WARNING:/ Only parse lengths of time from trusted input
1719−-- since an attacker could easily fill up the memory of the target
1720−-- system by specifying a scientific number with a big exponent like
1721−-- @1e1000000000@.
1726+-- | This instance includes a bounds check to prevent maliciously
1727+-- large inputs to fill up the memory of the target system. You can
1728+-- newtype 'Scientific' and provide your own instance using
1729+-- 'withScientific' if you want to allow larger inputs.
17221730instance FromJSON NominalDiffTime where
1723− parseJSON = withScientific "NominalDiffTime" $ pure . realToFrac
1731+ parseJSON = withBoundedScientific "NominalDiffTime" $ pure . realToFrac
17241732 {-# INLINE parseJSON #-}
17251733
17261734
1727−-- | /WARNING:/ Only parse lengths of time from trusted input
1728−-- since an attacker could easily fill up the memory of the target
1729−-- system by specifying a scientific number with a big exponent like
1730−-- @1e1000000000@.
1735+-- | This instance includes a bounds check to prevent maliciously
1736+-- large inputs to fill up the memory of the target system. You can
1737+-- newtype 'Scientific' and provide your own instance using
1738+-- 'withScientific' if you want to allow larger inputs.
17311739instance FromJSON DiffTime where
1732− parseJSON = withScientific "DiffTime" $ pure . realToFrac
1740+ parseJSON = withBoundedScientific "DiffTime" $ pure . realToFrac
17331741 {-# INLINE parseJSON #-}
17341742
17351743-------------------------------------------------------------------------------
aeson.cabal+1−1
@@ -1,5 +1,5 @@
11name: aeson
2−version: 1.3.1.1
2+version: 1.4.0.0
33license: BSD3
44license-file: LICENSE
55category: Text, Web, JSON
changelog.md+16−1
@@ -1,9 +1,24 @@
11For the latest version of this document, please see [https://github.com/bos/aeson/blob/master/changelog.md](https://github.com/bos/aeson/blob/master/changelog.md).
22
3−### 1.4.0.0 (upcoming)
3+### 1.4.0.0
44
5+This release introduces bounds on the size of `Scientific` numbers when they are converted to other arbitrary precision types that do not represent them efficiently in memory.
6+
7+This means that trying to decode a number such as `1e1000000000` into an `Integer` will now fail instead of using a lot of memory. If you need to represent large numbers you can add a newtype (preferably over `Scientific`) and providing a parser using `withScientific`.
8+
9+The following instances are affected by this:
10+* `FromJSON Natural`
11+* `FromJSONKey Natural`
12+* `FromJSON Integer`
13+* `FromJSONKey Integer`
14+* `FromJSON NominalDiffTime`
15+
16+For the same reasons the following instances & functions have been removed:
17+* Remove `FromJSON Data.Attoparsec.Number` instance. Note that `Data.Attoparsec.Number` is deprecated.
518* Remove deprecated `withNumber`, use `withScientific` instead.
619
20+Finally, encoding integral values with large exponents now uses scientific notation, this saves space for large numbers.
21+
722### 1.3.1.1
823
924* Catch 0 denominators when parsing Ratio
tests/UnitTests.hs+37
@@ -31,9 +31,11 @@ import Data.Aeson.Types (Options(..), Result(Success), ToJSON(..), Value(Null),
3131import Data.Char (toUpper)
3232import Data.Either.Compat (isLeft, isRight)
3333import Data.Hashable (hash)
34+import Data.HashMap.Strict (HashMap)
3435import Data.List (sort)
3536import Data.Maybe (fromMaybe)
3637import Data.Sequence (Seq)
38+import Data.Scientific (Scientific, scientific)
3739import Data.Tagged (Tagged(..))
3840import Data.Text (Text)
3941import Data.Time (UTCTime)
@@ -41,6 +43,7 @@ import Data.Time.Format (parseTime)
4143import Data.Time.Locale.Compat (defaultTimeLocale)
4244import GHC.Generics (Generic)
4345import Instances ()
46+import Numeric.Natural (Natural)
4447import System.Directory (getDirectoryContents)
4548import System.FilePath ((</>), takeExtension, takeFileName)
4649import Test.Tasty (TestTree, testGroup)
@@ -100,6 +103,10 @@ tests = testGroup "unit" [
100103 , testCase "withEmbeddedJSON" withEmbeddedJSONTest
101104 , testCase "SingleFieldCon" singleFieldCon
102105 , testCase "Ratio with denominator 0" ratioDenominator0
106+ , testCase "Big scientific exponent" bigScientificExponent
107+ , testCase "Big integer decoding" bigIntegerDecoding
108+ , testCase "Big natural decading" bigNaturalDecoding
109+ , testCase "Big integer key decoding" bigIntegerKeyDecoding
103110 ]
104111
105112roundTripCamel :: String -> Assertion
@@ -552,6 +559,36 @@ ratioDenominator0 =
552559 (Left "Error in $: Ratio denominator was 0")
553560 (eitherDecode "{ \"numerator\": 1, \"denominator\": 0 }" :: Either String Rational)
554561
562+bigScientificExponent :: Assertion
563+bigScientificExponent =
564+ assertEqual "Encoding an integral scientific with a large exponent should normalize it"
565+ "1.0e2000"
566+ (encode (scientific 1 2000 :: Scientific))
567+
568+bigIntegerDecoding :: Assertion
569+bigIntegerDecoding =
570+ assertEqual "Decoding an Integer with a large exponent should fail"
571+ (Left "Error in $: expected a number with exponent <= 1024, encountered Number")
572+ ((eitherDecode :: L.ByteString -> Either String Integer) "1e2000")
573+
574+bigNaturalDecoding :: Assertion
575+bigNaturalDecoding =
576+ assertEqual "Decoding a Natural with a large exponent should fail"
577+ (Left "Error in $: expected a number with exponent <= 1024, encountered Number")
578+ ((eitherDecode :: L.ByteString -> Either String Integer) "1e2000")
579+
580+bigIntegerKeyDecoding :: Assertion
581+bigIntegerKeyDecoding =
582+ assertEqual "Decoding an Integer key with a large exponent should fail"
583+ (Left "Error in $['1e2000']: expected a number with exponent <= 1024, encountered Number")
584+ ((eitherDecode :: L.ByteString -> Either String (HashMap Integer Value)) "{ \"1e2000\": null }")
585+
586+bigNaturalKeyDecoding :: Assertion
587+bigNaturalKeyDecoding =
588+ assertEqual "Decoding an Integer key with a large exponent should fail"
589+ (Left "Error in $['1e2000']: expected a number with exponent <= 1024, encountered Number")
590+ ((eitherDecode :: L.ByteString -> Either String (HashMap Natural Value)) "{ \"1e2000\": null }")
591+
555592deriveJSON defaultOptions{omitNothingFields=True} ''MyRecord
556593
557594deriveToJSON defaultOptions ''Foo