on master help sign in
Add an attoparsec based UTC time decoder (#197)

Replace the old 'parseTime' based UTCTime parser with an attoparsec based parser. This is roughly 5x faster than the old one. It also allows decoding some timestamp formats that the old version rejected. The new version still requires all timestamps to specify a timezone when decoding to a UTCTime type. Also add a few unit tests for both what should be accepted and also for things that should be rejected. A lot of this code was bummed from the postgresql-simple library - thus credits to @lpsmith too!

commitb0760435c163330d90aa4ebd76e85fa377eda90b authorJanne Hellsten <jjhellst@gmail.com> dateSun 12 Jul 2015, 22:22 (11 years ago) parent09b2ddc
2 files changed+167−9
Data/Aeson/Types/Instances.hs+90−7 tests/UnitTests.hs+77−2
Data/Aeson/Types/Instances.hs+90−7
@@ -1,6 +1,7 @@
1−{-# LANGUAGE CPP, DeriveDataTypeable, FlexibleContexts, FlexibleInstances,
2− GeneralizedNewtypeDeriving, IncoherentInstances, OverlappingInstances,
3− OverloadedStrings, UndecidableInstances, ViewPatterns #-}
1+{-# LANGUAGE CPP, BangPatterns, DeriveDataTypeable, FlexibleContexts,
2+ FlexibleInstances, GeneralizedNewtypeDeriving, IncoherentInstances,
3+ OverlappingInstances, OverloadedStrings, UndecidableInstances,
4+ ViewPatterns #-}
45{-# LANGUAGE DefaultSignatures #-}
56{-# OPTIONS_GHC -fno-warn-orphans #-}
67
@@ -55,9 +56,11 @@ module Data.Aeson.Types.Instances
5556 ) where
5657
5758import Control.Applicative ((<$>), (<*>), (<|>), pure, empty)
59+import Control.Monad (when)
5860import qualified Data.Aeson.Encode.Builder as E
5961import qualified Data.ByteString.Builder as B
6062import Data.Aeson.Functions
63+import Data.Bits ((.&.))
6164import Data.Monoid ((<>), mempty)
6265import Data.Aeson.Encode.Functions (brackets, builder, foldable, list)
6366import Data.Aeson.Types.Class
@@ -65,6 +68,8 @@ import Data.Aeson.Types.Internal
6568import Data.Scientific (Scientific)
6669import qualified Data.Scientific as Scientific (coefficient, base10Exponent, fromFloatDigits, toRealFloat)
6770import Data.Attoparsec.Number (Number(..))
71+import qualified Data.Attoparsec.Text as A
72+import Data.Char (isDigit, ord)
6873import Data.Fixed
6974import Data.Foldable (Foldable, toList)
7075import Data.Functor.Identity (Identity(..))
@@ -74,8 +79,10 @@ import Data.Maybe (fromMaybe)
7479import Data.Monoid (Dual(..), First(..), Last(..))
7580import Data.Ratio (Ratio, (%), numerator, denominator)
7681import Data.Text (Text, pack, unpack)
77−import Data.Time (UTCTime, ZonedTime(..), TimeZone(..))
82+import Data.Time (UTCTime(..), ZonedTime(..), TimeZone(..), Day, utc)
83+import Data.Time.Calendar (fromGregorianValid, addDays)
7884import Data.Time.Format (FormatTime, formatTime, parseTime)
85+import Data.Time.LocalTime (TimeOfDay, makeTimeOfDayValid, minutesToTimeZone, timeOfDayToTime, localToUTCTimeOfDay)
7986import Data.Traversable as Tr (sequence, traverse)
8087import Data.Vector (Vector)
8188import Data.Word (Word, Word8, Word16, Word32, Word64)
@@ -767,11 +774,87 @@ utcTime :: UTCTime -> String
767774utcTime t = formatTime defaultTimeLocale format t
768775 where format = "%FT%T." ++ formatSubseconds t ++ "Z"
769776
777+getDay :: A.Parser Day
778+getDay = do
779+ yearStr <- A.takeWhile isDigit
780+ when (T.length yearStr < 4) (fail "year must consist of at least 4 digits")
781+
782+ let !year = toNum yearStr
783+ _ <- A.char '-'
784+ month <- digits "month"
785+ _ <- A.char '-'
786+ day <- digits "day"
787+
788+ case fromGregorianValid year month day of
789+ Nothing -> fail "invalid date"
790+ Just x -> return $! x
791+
792+toNum :: Num n => T.Text -> n
793+toNum = T.foldl' (\a c -> 10*a + digit c) 0
794+{-# INLINE toNum #-}
795+
796+digit :: Num n => Char -> n
797+digit c = fromIntegral (ord c .&. 0x0f)
798+{-# INLINE digit #-}
799+
800+digits :: Num n => String -> A.Parser n
801+digits msg = do
802+ x <- A.anyChar
803+ y <- A.anyChar
804+ if isDigit x && isDigit y
805+ then return $! (10 * digit x + digit y)
806+ else fail (msg ++ " is not 2 digits")
807+{-# INLINE digits #-}
808+
809+decimal :: Fractional a => T.Text -> a
810+decimal str = toNum str / 10^(T.length str)
811+{-# INLINE decimal #-}
812+
813+getTimeOfDay :: A.Parser TimeOfDay
814+getTimeOfDay = do
815+ hour <- digits "hours"
816+ _ <- A.char ':'
817+ minute <- digits "minutes"
818+ -- Allow omission of seconds. If seconds is omitted, don't try to
819+ -- parse the sub-second part.
820+ (sec,subsec)
821+ <- ((,) <$> (A.char ':' *> digits "seconds") <*> fract) <|> pure (0,0)
822+
823+ let !picos' = sec + subsec
824+
825+ case makeTimeOfDayValid hour minute picos' of
826+ Nothing -> fail "invalid time of day"
827+ Just x -> return $! x
828+
829+ where
830+ fract =
831+ (A.char '.' *> (decimal <$> A.takeWhile1 isDigit)) <|> pure 0
832+
833+getTimeZone :: A.Parser TimeZone
834+getTimeZone = do
835+ sign <- A.satisfy (\c -> c == '+' || c == '-')
836+ hours <- digits "timezone"
837+ mins <- (A.char ':' *> digits "timezone minutes") <|> pure 0
838+ let !absset = 60 * hours + mins
839+ !offset = if sign == '+' then absset else -absset
840+ return $! minutesToTimeZone offset
841+
842+getUTCTime :: A.Parser UTCTime
843+getUTCTime = do
844+ day <- getDay
845+ _ <- A.char ' ' <|> A.char 'T'
846+ time <- getTimeOfDay
847+ zone <- getTimeZone <|> (A.char 'Z' *> pure utc)
848+ let (!dayDelta,!time') = localToUTCTimeOfDay zone time
849+ let !day' = addDays dayDelta day
850+ let !time'' = timeOfDayToTime time'
851+ return (UTCTime day' time'')
852+
770853instance FromJSON UTCTime where
771854 parseJSON = withText "UTCTime" $ \t ->
772− case parseTime defaultTimeLocale "%FT%T%QZ" (unpack t) of
773− Just d -> pure d
774− _ -> fail "could not parse ISO-8601 date"
855+ case A.parseOnly (getUTCTime <* A.endOfInput) t of
856+ Right d -> pure d
857+ Left msg -> fail ("could not parse ISO-8601 date: " ++ msg)
775858
776859parseJSONElemAtIndex :: FromJSON a => Int -> Vector Value -> Parser a
777860parseJSONElemAtIndex idx ary = parseJSON (V.unsafeIndex ary idx) <?> Index idx
tests/UnitTests.hs+77−2
@@ -1,12 +1,14 @@
1−{-# LANGUAGE DeriveGeneric #-}
1+{-# LANGUAGE CPP, DeriveGeneric, OverloadedStrings, ScopedTypeVariables #-}
22
33module UnitTests (ioTests, tests) where
44
55import Control.Monad (forM)
6−import Data.Aeson (eitherDecode, encode, genericToJSON, genericToEncoding)
6+import Data.Aeson (decode, eitherDecode, encode, genericToJSON, genericToEncoding)
77import Data.Aeson.Encode (encodeToTextBuilder)
88import Data.Aeson.Types (ToJSON(..), Value, camelTo, defaultOptions)
99import Data.Char (toUpper)
10+import Data.Time (UTCTime)
11+import Data.Time.Format (parseTime)
1012import GHC.Generics (Generic)
1113import Test.Framework (Test, testGroup)
1214import Test.Framework.Providers.HUnit (testCase)
@@ -14,6 +16,14 @@ import Test.HUnit (Assertion, assertFailure, assertEqual)
1416import qualified Data.ByteString.Lazy.Char8 as L
1517import qualified Data.Text.Lazy.Builder as TLB
1618import qualified Data.Text.Lazy.Encoding as TLE
19+import qualified Data.Text.Lazy as LT
20+import qualified Data.Text.Lazy.Encoding as LT
21+
22+#if MIN_VERSION_time(1,5,0)
23+import Data.Time.Format (defaultTimeLocale)
24+#else
25+import System.Locale (defaultTimeLocale)
26+#endif
1727
1828tests :: Test
1929tests = testGroup "unit" [
@@ -25,6 +35,10 @@ tests = testGroup "unit" [
2535 , testGroup "encoding" [
2636 testCase "goodProducer" $ goodProducer
2737 ]
38+ , testGroup "utctime" [
39+ testCase "good" $ utcTimeGood
40+ , testCase "bad" $ utcTimeBad
41+ ]
2842 ]
2943
3044roundTripCamel :: String -> Assertion
@@ -57,6 +71,67 @@ goodProducer = assertEqual "partial encoding should not explode on undefined"
5771 , wibbleInt = undefined
5872 }
5973
74+-- Test decoding various UTC time formats
75+--
76+-- Note: the incomplete pattern matches for UTCTimes are completely
77+-- intentional. The test expects these parses to succeed. If the
78+-- pattern matches fails, there's a bug in either the test or in aeson
79+-- and needs to be investigated.
80+utcTimeGood :: Assertion
81+utcTimeGood = do
82+ let ts1 = "2015-01-01T12:13:00.00Z" :: LT.Text
83+ let ts2 = "2015-01-01T12:13:00Z" :: LT.Text
84+ -- 'T' between date and time is not required, can be space
85+ let ts3 = "2015-01-03 12:13:00.00Z" :: LT.Text
86+ let ts4 = "2015-01-03 12:13:00.125Z" :: LT.Text
87+ let (Just (t1 :: UTCTime)) = parseWithAeson ts1
88+ let (Just (t2 :: UTCTime)) = parseWithAeson ts2
89+ let (Just (t3 :: UTCTime)) = parseWithAeson ts3
90+ let (Just (t4 :: UTCTime)) = parseWithAeson ts4
91+ assertEqual "utctime" (parseWithRead "%FT%T%QZ" ts1) t1
92+ assertEqual "utctime" (parseWithRead "%FT%T%QZ" ts2) t2
93+ assertEqual "utctime" (parseWithRead "%F %T%QZ" ts3) t3
94+ assertEqual "utctime" (parseWithRead "%F %T%QZ" ts4) t4
95+ -- timezones. Both +HH and +HH:MM are allowed for timezone offset
96+ let ts5 = "2015-01-01T12:30:00.00+00" :: LT.Text
97+ let ts6 = "2015-01-01T12:30:00.00+01:15" :: LT.Text
98+ let ts7 = "2015-01-01T12:30:00.00-02" :: LT.Text
99+ let ts8 = "2015-01-01T22:00:00.00-03" :: LT.Text
100+ let (Just (t5 :: UTCTime)) = parseWithAeson ts5
101+ let (Just (t6 :: UTCTime)) = parseWithAeson ts6
102+ let (Just (t7 :: UTCTime)) = parseWithAeson ts7
103+ let (Just (t8 :: UTCTime)) = parseWithAeson ts8
104+ assertEqual "utctime" (parseWithRead "%FT%T%QZ" "2015-01-01T12:30:00.00Z") t5
105+ assertEqual "utctime" (parseWithRead "%FT%T%QZ" "2015-01-01T11:15:00.00Z") t6
106+ assertEqual "utctime" (parseWithRead "%FT%T%QZ" "2015-01-01T14:30:00Z") t7
107+ -- ts8 wraps around to the next day in UTC
108+ assertEqual "utctime" (parseWithRead "%FT%T%QZ" "2015-01-02T01:00:00Z") t8
109+ where
110+ parseWithRead :: String -> LT.Text -> UTCTime
111+ parseWithRead f s =
112+ case parseTime defaultTimeLocale f . LT.unpack $ s of
113+ Nothing -> error "parseTime input malformed"
114+ Just t -> t
115+ parseWithAeson :: LT.Text -> Maybe UTCTime
116+ parseWithAeson s = decode . LT.encodeUtf8 $ (LT.concat ["\"", s, "\""])
117+
118+-- Test that a few non-timezone qualified timestamp formats get
119+-- rejected if decoding to UTCTime.
120+utcTimeBad :: Assertion
121+utcTimeBad = do
122+ verifyFailParse "2000-01-01T12:13:00" -- missing Zulu time not allowed (some TZ required)
123+ verifyFailParse "2000-01-01 12:13:00" -- missing Zulu time not allowed (some TZ required)
124+ verifyFailParse "2000-01-01" -- date only not OK
125+ verifyFailParse "2000-01-01Z" -- date only not OK
126+ verifyFailParse "2015-01-01T12:30:00.00+00Z" -- no Zulu if offset given
127+ verifyFailParse "2015-01-01T12:30:00.00+00:00Z" -- no Zulu if offset given
128+ verifyFailParse "2015-01-03 12:13:00.Z" -- decimal at the end but no digits
129+ where
130+ verifyFailParse (s :: LT.Text) =
131+ let (dec :: Maybe UTCTime) = decode . LT.encodeUtf8 $ (LT.concat ["\"", s, "\""]) in
132+ assertEqual "verify failure" Nothing dec
133+
134+
60135------------------------------------------------------------------------------
61136-- Comparison between bytestring and text encoders
62137------------------------------------------------------------------------------