git.iko.soy/servant-quickcheck
on master help sign in
Resolve merge conflicts with unbiasedGeneration merge
commitf3b4fcf7a902202ad73f972b6d6ca9d9d632549e authorErik Aker <eraker@gmail.com> dateSat 14 Oct 2017, 08:16 (9 years ago) parent0e23a2e66ce509
3 files changed+128−53
src/Servant/QuickCheck/Internal/HasGenRequest.hs+55−31 src/Servant/QuickCheck/Internal/QuickCheck.hs+3−3 test/Servant/QuickCheck/InternalSpec.hs+70−19
src/Servant/QuickCheck/Internal/HasGenRequest.hs+55−31
@@ -14,106 +14,130 @@ import Prelude.Compat
1414import Servant
1515import Servant.API.ContentTypes (AllMimeRender (..))
1616import Servant.Client (BaseUrl (..), Scheme (..))
17−import Test.QuickCheck (Arbitrary (..), Gen, elements, oneof)
17+import Test.QuickCheck (Arbitrary (..), Gen, elements, frequency)
1818
1919import qualified Data.ByteString as BS
2020import qualified Data.ByteString.Internal as BS (c2w)
2121
2222
23+-- -----------------------------------------------------------------------------
24+-- runGenRequest
25+
26+-- | This function returns a QuickCheck `Gen a` when passed a servant API value,
27+-- typically a `Proxy API`. The generator returned is a function
28+-- that accepts a `BaseUrl` and returns a `Request`, which can then be used
29+-- to issue network requests. This `Gen` type makes it easier to compare distinct
30+-- APIs across different `BaseUrl`s.
31+runGenRequest :: HasGenRequest a => Proxy a -> Gen (BaseUrl -> Request)
32+runGenRequest = snd . genRequest
33+
34+
35+-- -----------------------------------------------------------------------------
36+-- HasGenRequest
37+
38+-- | This is the core Servant-Quickcheck generator, which, when given a `Proxy API`
39+-- will return a pair of `Int` and `Gen a`, where `a` is a function from
40+-- `BaseUrl` to a `Network.Http.Client.Request`. The `Int` is a weight for the
41+-- QuickCheck `frequency` function which ensures a random distribution across
42+-- all endpoints in an API.
2343class HasGenRequest a where
24− genRequest :: Proxy a -> Gen (BaseUrl -> Request)
44+ genRequest :: Proxy a -> (Int, Gen (BaseUrl -> Request))
45+
2546
2647instance (HasGenRequest a, HasGenRequest b) => HasGenRequest (a :<|> b) where
2748 genRequest _
28− = oneof [ genRequest (Proxy :: Proxy a)
29− , genRequest (Proxy :: Proxy b)
30− ]
49+ = (lf + rf, frequency [l, r])
50+ where
51+ l@(lf, _) = genRequest (Proxy :: Proxy a)
52+ r@(rf, _) = genRequest (Proxy :: Proxy b)
53+
3154
3255instance (KnownSymbol path, HasGenRequest b) => HasGenRequest (path :> b) where
33− genRequest _ = do
56+ genRequest _ = (oldf, do
3457 old' <- old
3558 return $ \burl -> let r = old' burl
3659 oldPath = path r
3760 oldPath' = BS.dropWhile (== BS.c2w '/') oldPath
3861 paths = filter (not . BS.null) [new, oldPath']
39− in r { path = "/" <> BS.intercalate "/" paths }
62+ in r { path = "/" <> BS.intercalate "/" paths })
4063 where
41− old = genRequest (Proxy :: Proxy b)
64+ (oldf, old) = genRequest (Proxy :: Proxy b)
4265 new = cs $ symbolVal (Proxy :: Proxy path)
4366
67+
4468instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
4569 => HasGenRequest (Capture x c :> b) where
46− genRequest _ = do
70+ genRequest _ = (oldf, do
4771 old' <- old
4872 new' <- toUrlPiece <$> new
49− return $ \burl -> let r = old' burl in r { path = cs new' <> path r }
73+ return $ \burl -> let r = old' burl in r { path = cs new' <> path r })
5074 where
51− old = genRequest (Proxy :: Proxy b)
75+ (oldf, old) = genRequest (Proxy :: Proxy b)
5276 new = arbitrary :: Gen c
5377
5478#if MIN_VERSION_servant(0,8,0)
5579instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
5680 => HasGenRequest (CaptureAll x c :> b) where
57− genRequest _ = do
81+ genRequest _ = (oldf, do
5882 old' <- old
5983 new' <- fmap (cs . toUrlPiece) <$> new
6084 let new'' = BS.intercalate "/" new'
61− return $ \burl -> let r = old' burl in r { path = new'' <> path r }
85+ return $ \burl -> let r = old' burl in r { path = new'' <> path r })
6286 where
63− old = genRequest (Proxy :: Proxy b)
87+ (oldf, old) = genRequest (Proxy :: Proxy b)
6488 new = arbitrary :: Gen [c]
6589#endif
6690
6791instance (Arbitrary c, KnownSymbol h, HasGenRequest b, ToHttpApiData c)
6892 => HasGenRequest (Header h c :> b) where
69− genRequest _ = do
93+ genRequest _ = (oldf, do
7094 old' <- old
7195 new' <- toUrlPiece <$> new
7296 return $ \burl -> let r = old' burl in r {
73− requestHeaders = (hdr, cs new') : requestHeaders r }
97+ requestHeaders = (hdr, cs new') : requestHeaders r })
7498 where
75− old = genRequest (Proxy :: Proxy b)
99+ (oldf, old) = genRequest (Proxy :: Proxy b)
76100 hdr = fromString $ symbolVal (Proxy :: Proxy h)
77101 new = arbitrary :: Gen c
78102
79103instance (AllMimeRender x c, Arbitrary c, HasGenRequest b)
80104 => HasGenRequest (ReqBody x c :> b) where
81− genRequest _ = do
105+ genRequest _ = (oldf, do
82106 old' <- old
83107 new' <- new
84108 (ct, bd) <- elements $ allMimeRender (Proxy :: Proxy x) new'
85109 return $ \burl -> let r = old' burl in r {
86110 requestBody = RequestBodyLBS bd
87111 , requestHeaders = ("Content-Type", renderHeader ct) : requestHeaders r
88− }
112+ })
89113 where
90− old = genRequest (Proxy :: Proxy b)
114+ (oldf, old) = genRequest (Proxy :: Proxy b)
91115 new = arbitrary :: Gen c
92116
93117instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
94118 => HasGenRequest (QueryParam x c :> b) where
95− genRequest _ = do
119+ genRequest _ = (oldf, do
96120 new' <- new
97121 old' <- old
98122 return $ \burl -> let r = old' burl
99123 newExpr = param <> "=" <> cs (toQueryParam new')
100124 qs = queryString r in r {
101− queryString = if BS.null qs then newExpr else newExpr <> "&" <> qs }
125+ queryString = if BS.null qs then newExpr else newExpr <> "&" <> qs })
102126 where
103− old = genRequest (Proxy :: Proxy b)
127+ (oldf, old) = genRequest (Proxy :: Proxy b)
104128 param = cs $ symbolVal (Proxy :: Proxy x)
105129 new = arbitrary :: Gen c
106130
107131instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
108132 => HasGenRequest (QueryParams x c :> b) where
109− genRequest _ = do
133+ genRequest _ = (oldf, do
110134 new' <- new
111135 old' <- old
112136 return $ \burl -> let r = old' burl in r {
113137 queryString = queryString r
114− <> if length new' > 0 then fold (toParam <$> new') else ""}
138+ <> if length new' > 0 then fold (toParam <$> new') else ""})
115139 where
116− old = genRequest (Proxy :: Proxy b)
140+ (oldf, old) = genRequest (Proxy :: Proxy b)
117141 param = cs $ symbolVal (Proxy :: Proxy x)
118142 new = arbitrary :: Gen [c]
119143 toParam c = param <> "[]=" <> cs (toQueryParam c)
@@ -121,23 +145,23 @@ instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
121145
122146instance (KnownSymbol x, HasGenRequest b)
123147 => HasGenRequest (QueryFlag x :> b) where
124− genRequest _ = do
148+ genRequest _ = (oldf, do
125149 old' <- old
126150 return $ \burl -> let r = old' burl
127151 qs = queryString r in r {
128− queryString = if BS.null qs then param else param <> "&" <> qs }
152+ queryString = if BS.null qs then param else param <> "&" <> qs })
129153 where
130− old = genRequest (Proxy :: Proxy b)
154+ (oldf, old) = genRequest (Proxy :: Proxy b)
131155 param = cs $ symbolVal (Proxy :: Proxy x)
132156
133157instance (ReflectMethod method)
134158 => HasGenRequest (Verb (method :: k) (status :: Nat) (cts :: [*]) a) where
135− genRequest _ = return $ \burl -> defaultRequest
159+ genRequest _ = (1, return $ \burl -> defaultRequest
136160 { host = cs $ baseUrlHost burl
137161 , port = baseUrlPort burl
138162 , secure = baseUrlScheme burl == Https
139163 , method = reflectMethod (Proxy :: Proxy method)
140− }
164+ })
141165
142166instance (HasGenRequest a) => HasGenRequest (RemoteHost :> a) where
143167 genRequest _ = genRequest (Proxy :: Proxy a)
src/Servant/QuickCheck/Internal/QuickCheck.hs+3−3
@@ -70,7 +70,7 @@ withServantServerAndContext api ctx server t
7070serversEqual :: HasGenRequest a =>
7171 Proxy a -> BaseUrl -> BaseUrl -> Args -> ResponseEquality LBS.ByteString -> Expectation
7272serversEqual api burl1 burl2 args req = do
73− let reqs = (\f -> (f burl1, f burl2)) <$> genRequest api
73+ let reqs = (\f -> (f burl1, f burl2)) <$> runGenRequest api
7474 -- This MVar stuff is clunky! But there doesn't seem to be an easy way to
7575 -- return results when a test fails, since an exception is throw.
7676 deetsMVar <- newMVar $ error "should not be called"
@@ -111,7 +111,7 @@ serversEqual api burl1 burl2 args req = do
111111serverSatisfies :: (HasGenRequest a) =>
112112 Proxy a -> BaseUrl -> Args -> Predicates -> Expectation
113113serverSatisfies api burl args preds = do
114− let reqs = ($ burl) <$> genRequest api
114+ let reqs = ($ burl) <$> runGenRequest api
115115 deetsMVar <- newMVar $ error "should not be called"
116116 r <- quickCheckWithResult args { chatty = False } $ monadicIO $ forAllM reqs $ \req -> do
117117 v <- run $ finishPredicates preds (noCheckStatus req) defManager
@@ -131,7 +131,7 @@ serverSatisfies api burl args preds = do
131131serverDoesntSatisfy :: (HasGenRequest a) =>
132132 Proxy a -> BaseUrl -> Args -> Predicates -> Expectation
133133serverDoesntSatisfy api burl args preds = do
134− let reqs = ($ burl) <$> genRequest api
134+ let reqs = ($ burl) <$> runGenRequest api
135135 r <- quickCheckWithResult args $ monadicIO $ forAllM reqs $ \req -> do
136136 v <- run $ finishPredicates preds (noCheckStatus req) defManager
137137 assert $ not $ null v
test/Servant/QuickCheck/InternalSpec.hs+70−19
@@ -1,20 +1,24 @@
11{-# LANGUAGE CPP #-}
22module Servant.QuickCheck.InternalSpec (spec) where
33
4−import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
5−import Control.Exception
6−import Control.Monad.IO.Class (liftIO)
7−import qualified Data.ByteString as BS
8−import qualified Data.ByteString.Char8 as C
4+
5+import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
6+import Control.Exception (SomeException)
7+import Control.Monad (replicateM)
8+import Control.Monad.IO.Class (liftIO)
9+import qualified Data.ByteString as BS
10+import qualified Data.ByteString.Char8 as C
11+import Data.Maybe (fromJust)
912import Prelude.Compat
1013import Servant
11−import Test.Hspec (Spec, context, describe, it, shouldBe,
12− shouldContain)
13−import Test.Hspec.Core.Spec (Arg, Example, Result (..),
14− defaultParams, safeEvaluateExample)
15−import Test.QuickCheck.Gen (unGen)
16−import Test.QuickCheck.Random (mkQCGen)
17−import Network.HTTP.Client (queryString, path)
14+import Test.Hspec (Spec, context, describe, it, shouldBe,
15+ shouldContain)
16+import Test.Hspec.Core.Spec (Arg, Example, Result (..),
17+ defaultParams, safeEvaluateExample)
18+import Test.QuickCheck.Gen (unGen, generate)
19+import Test.QuickCheck.Random (mkQCGen)
20+import Network.HTTP.Client (queryString, path)
21+
1822
1923#if MIN_VERSION_servant(0,8,0)
2024import Servant.API.Internal.Test.ComprehensiveAPI (comprehensiveAPIWithoutRaw)
@@ -24,7 +28,8 @@ import Servant.API.Internal.Test.ComprehensiveAPI (ComprehensiveAPI,
2428#endif
2529
2630import Servant.QuickCheck
27−import Servant.QuickCheck.Internal (genRequest, serverDoesntSatisfy)
31+import Servant.QuickCheck.Internal (genRequest, runGenRequest, serverDoesntSatisfy)
32+
2833
2934spec :: Spec
3035spec = do
@@ -36,6 +41,7 @@ spec = do
3641 queryParamsSpec
3742 queryFlagsSpec
3843 deepPathSpec
44+ unbiasedGenerationSpec
3945
4046serversEqualSpec :: Spec
4147serversEqualSpec = describe "serversEqual" $ do
@@ -46,7 +52,6 @@ serversEqualSpec = describe "serversEqual" $ do
4652 serversEqual api burl1 burl2 args bodyEquality
4753
4854 context "when servers are not equal" $ do
49−
5055 it "provides the failing responses in the error message" $ do
5156 Right (Failure _ err) <- withServantServer api2 server2 $ \burl1 ->
5257 withServantServer api2 server3 $ \burl2 -> do
@@ -77,11 +82,12 @@ serverSatisfiesSpec = describe "serverSatisfies" $ do
7782
7883 it "fails with informative error messages" $ do
7984 Right (Failure _ err) <- withServantServerAndContext api ctx server $ \burl -> do
80− safeEvalExample $ serverSatisfies api burl args (getsHaveCacheControlHeader <%> mempty)
81− show err `shouldContain` "getsHaveCacheControlHeader"
85+ safeEvalExample $ serverSatisfies api burl args (notAllowedContainsAllowHeader <%> mempty)
86+ show err `shouldContain` "notAllowedContainsAllowHeader"
8287 show err `shouldContain` "Headers"
8388 show err `shouldContain` "Body"
8489
90+
8591onlyJsonObjectSpec :: Spec
8692onlyJsonObjectSpec = describe "onlyJsonObjects" $ do
8793
@@ -121,7 +127,7 @@ deepPathSpec = describe "Path components" $ do
121127 it "are separated by slashes, without a trailing slash" $ do
122128 let rng = mkQCGen 0
123129 burl = BaseUrl Http "localhost" 80 ""
124− gen = genRequest deepAPI
130+ gen = runGenRequest deepAPI
125131 req = (unGen gen rng 0) burl
126132 path req `shouldBe` ("/one/two/three")
127133
@@ -132,7 +138,7 @@ queryParamsSpec = describe "QueryParams" $ do
132138 it "reduce to an HTTP query string correctly" $ do
133139 let rng = mkQCGen 0
134140 burl = BaseUrl Http "localhost" 80 ""
135− gen = genRequest paramsAPI
141+ gen = runGenRequest paramsAPI
136142 req = (unGen gen rng 0) burl
137143 qs = C.unpack $ queryString req
138144 qs `shouldBe` "one=_&two=_"
@@ -143,11 +149,33 @@ queryFlagsSpec = describe "QueryFlags" $ do
143149 it "reduce to an HTTP query string correctly" $ do
144150 let rng = mkQCGen 0
145151 burl = BaseUrl Http "localhost" 80 ""
146− gen = genRequest flagsAPI
152+ gen = runGenRequest flagsAPI
147153 req = (unGen gen rng 0) burl
148154 qs = C.unpack $ queryString req
149155 qs `shouldBe` "one&two"
150156
157+makeRandomRequest :: Proxy LargeAPI -> BaseUrl -> IO Integer
158+makeRandomRequest large burl = do
159+ req <- generate $ runGenRequest large
160+ pure $ fst . fromJust . C.readInteger . C.drop 1 . path $ req burl
161+
162+
163+unbiasedGenerationSpec :: Spec
164+unbiasedGenerationSpec = describe "Unbiased Generation of requests" $
165+
166+ it "frequency paired with generated endpoint should be more randomly distributed" $ do
167+ let burl = BaseUrl Http "localhost" 80 ""
168+ let runs = 10000 :: Double
169+ someRequests <- replicateM 10000 (makeRandomRequest largeApi burl)
170+ let mean = (sum $ map fromIntegral someRequests) / runs
171+ let variancer x = let ix = fromIntegral x in (ix - mean) * (ix - mean)
172+ let variance = (sum $ map variancer someRequests) / runs - 1
173+ -- mean should be around 8.5. If this fails, we likely need more runs (or there's a bug!)
174+ mean > 8 `shouldBe` True
175+ mean < 9 `shouldBe` True
176+ -- Std dev is likely around 4. Variance is probably greater than 20.
177+ variance > 19.5 `shouldBe` True
178+
151179------------------------------------------------------------------------------
152180-- APIs
153181------------------------------------------------------------------------------
@@ -195,6 +223,29 @@ server2 = return $ return 1
195223server3 :: IO (Server API2)
196224server3 = return $ return 2
197225
226+
227+largeApi :: Proxy LargeAPI
228+largeApi = Proxy
229+
230+type LargeAPI
231+ = "1" :> Get '[JSON] Int
232+ :<|> "2" :> Get '[JSON] Int
233+ :<|> "3" :> Get '[JSON] Int
234+ :<|> "4" :> Get '[JSON] Int
235+ :<|> "5" :> Get '[JSON] Int
236+ :<|> "6" :> Get '[JSON] Int
237+ :<|> "7" :> Get '[JSON] Int
238+ :<|> "8" :> Get '[JSON] Int
239+ :<|> "9" :> Get '[JSON] Int
240+ :<|> "10" :> Get '[JSON] Int
241+ :<|> "11" :> Get '[JSON] Int
242+ :<|> "12" :> Get '[JSON] Int
243+ :<|> "13" :> Get '[JSON] Int
244+ :<|> "14" :> Get '[JSON] Int
245+ :<|> "15" :> Get '[JSON] Int
246+ :<|> "16" :> Get '[JSON] Int
247+
248+
198249type OctetAPI = Get '[OctetStream] BS.ByteString
199250
200251octetAPI :: Proxy OctetAPI