git.iko.soy/servant-quickcheck
on master help sign in
Unbias generation of requests. (#19)

* Unbias generation of requests: ...so that each endpoint is picked with the same frequency. Also, include a test for unbiased generation that measures mean and variance of results.

commit66ce50993f45d5fa813b8108adc8263a8f6d13c6 authorJulian Arni <jkarni@gmail.com> dateSat 14 Oct 2017, 07:53 (9 years ago) parente1a9db4
3 files changed+125−53
src/Servant/QuickCheck/Internal/HasGenRequest.hs+55−31 src/Servant/QuickCheck/Internal/QuickCheck.hs+3−3 test/Servant/QuickCheck/InternalSpec.hs+67−19
src/Servant/QuickCheck/Internal/HasGenRequest.hs+55−31
@@ -14,106 +14,130 @@ import Prelude.Compat
1414import Servant
1515import Servant.API.ContentTypes (AllMimeRender (..))
1616import Servant.Client (BaseUrl (..), Scheme (..))
17−import Test.QuickCheck (Arbitrary (..), Gen, elements, oneof)
17+import Test.QuickCheck (Arbitrary (..), Gen, elements, frequency)
1818
1919import qualified Data.ByteString as BS
2020import qualified Data.ByteString.Internal as BS (c2w)
2121
2222
23+-- -----------------------------------------------------------------------------
24+-- runGenRequest
25+
26+-- | This function returns a QuickCheck `Gen a` when passed a servant API value,
27+-- typically a `Proxy API`. The generator returned is a function
28+-- that accepts a `BaseUrl` and returns a `Request`, which can then be used
29+-- to issue network requests. This `Gen` type makes it easier to compare distinct
30+-- APIs across different `BaseUrl`s.
31+runGenRequest :: HasGenRequest a => Proxy a -> Gen (BaseUrl -> Request)
32+runGenRequest = snd . genRequest
33+
34+
35+-- -----------------------------------------------------------------------------
36+-- HasGenRequest
37+
38+-- | This is the core Servant-Quickcheck generator, which, when given a `Proxy API`
39+-- will return a pair of `Int` and `Gen a`, where `a` is a function from
40+-- `BaseUrl` to a `Network.Http.Client.Request`. The `Int` is a weight for the
41+-- QuickCheck `frequency` function which ensures a random distribution across
42+-- all endpoints in an API.
2343class HasGenRequest a where
24− genRequest :: Proxy a -> Gen (BaseUrl -> Request)
44+ genRequest :: Proxy a -> (Int, Gen (BaseUrl -> Request))
45+
2546
2647instance (HasGenRequest a, HasGenRequest b) => HasGenRequest (a :<|> b) where
2748 genRequest _
28− = oneof [ genRequest (Proxy :: Proxy a)
29− , genRequest (Proxy :: Proxy b)
30− ]
49+ = (lf + rf, frequency [l, r])
50+ where
51+ l@(lf, _) = genRequest (Proxy :: Proxy a)
52+ r@(rf, _) = genRequest (Proxy :: Proxy b)
53+
3154
3255instance (KnownSymbol path, HasGenRequest b) => HasGenRequest (path :> b) where
33− genRequest _ = do
56+ genRequest _ = (oldf, do
3457 old' <- old
3558 return $ \burl -> let r = old' burl
3659 oldPath = path r
3760 oldPath' = BS.dropWhile (== BS.c2w '/') oldPath
3861 paths = filter (not . BS.null) [new, oldPath']
39− in r { path = "/" <> BS.intercalate "/" paths }
62+ in r { path = "/" <> BS.intercalate "/" paths })
4063 where
41− old = genRequest (Proxy :: Proxy b)
64+ (oldf, old) = genRequest (Proxy :: Proxy b)
4265 new = cs $ symbolVal (Proxy :: Proxy path)
4366
67+
4468instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
4569 => HasGenRequest (Capture x c :> b) where
46− genRequest _ = do
70+ genRequest _ = (oldf, do
4771 old' <- old
4872 new' <- toUrlPiece <$> new
49− return $ \burl -> let r = old' burl in r { path = cs new' <> path r }
73+ return $ \burl -> let r = old' burl in r { path = cs new' <> path r })
5074 where
51− old = genRequest (Proxy :: Proxy b)
75+ (oldf, old) = genRequest (Proxy :: Proxy b)
5276 new = arbitrary :: Gen c
5377
5478#if MIN_VERSION_servant(0,8,0)
5579instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
5680 => HasGenRequest (CaptureAll x c :> b) where
57− genRequest _ = do
81+ genRequest _ = (oldf, do
5882 old' <- old
5983 new' <- fmap (cs . toUrlPiece) <$> new
6084 let new'' = BS.intercalate "/" new'
61− return $ \burl -> let r = old' burl in r { path = new'' <> path r }
85+ return $ \burl -> let r = old' burl in r { path = new'' <> path r })
6286 where
63− old = genRequest (Proxy :: Proxy b)
87+ (oldf, old) = genRequest (Proxy :: Proxy b)
6488 new = arbitrary :: Gen [c]
6589#endif
6690
6791instance (Arbitrary c, KnownSymbol h, HasGenRequest b, ToHttpApiData c)
6892 => HasGenRequest (Header h c :> b) where
69− genRequest _ = do
93+ genRequest _ = (oldf, do
7094 old' <- old
7195 new' <- toUrlPiece <$> new
7296 return $ \burl -> let r = old' burl in r {
73− requestHeaders = (hdr, cs new') : requestHeaders r }
97+ requestHeaders = (hdr, cs new') : requestHeaders r })
7498 where
75− old = genRequest (Proxy :: Proxy b)
99+ (oldf, old) = genRequest (Proxy :: Proxy b)
76100 hdr = fromString $ symbolVal (Proxy :: Proxy h)
77101 new = arbitrary :: Gen c
78102
79103instance (AllMimeRender x c, Arbitrary c, HasGenRequest b)
80104 => HasGenRequest (ReqBody x c :> b) where
81− genRequest _ = do
105+ genRequest _ = (oldf, do
82106 old' <- old
83107 new' <- new
84108 (ct, bd) <- elements $ allMimeRender (Proxy :: Proxy x) new'
85109 return $ \burl -> let r = old' burl in r {
86110 requestBody = RequestBodyLBS bd
87111 , requestHeaders = ("Content-Type", renderHeader ct) : requestHeaders r
88− }
112+ })
89113 where
90− old = genRequest (Proxy :: Proxy b)
114+ (oldf, old) = genRequest (Proxy :: Proxy b)
91115 new = arbitrary :: Gen c
92116
93117instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
94118 => HasGenRequest (QueryParam x c :> b) where
95− genRequest _ = do
119+ genRequest _ = (oldf, do
96120 new' <- new
97121 old' <- old
98122 return $ \burl -> let r = old' burl
99123 newExpr = param <> "=" <> cs (toQueryParam new')
100124 qs = queryString r in r {
101− queryString = if BS.null qs then newExpr else newExpr <> "&" <> qs }
125+ queryString = if BS.null qs then newExpr else newExpr <> "&" <> qs })
102126 where
103− old = genRequest (Proxy :: Proxy b)
127+ (oldf, old) = genRequest (Proxy :: Proxy b)
104128 param = cs $ symbolVal (Proxy :: Proxy x)
105129 new = arbitrary :: Gen c
106130
107131instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
108132 => HasGenRequest (QueryParams x c :> b) where
109− genRequest _ = do
133+ genRequest _ = (oldf, do
110134 new' <- new
111135 old' <- old
112136 return $ \burl -> let r = old' burl in r {
113137 queryString = queryString r
114− <> if length new' > 0 then fold (toParam <$> new') else ""}
138+ <> if length new' > 0 then fold (toParam <$> new') else ""})
115139 where
116− old = genRequest (Proxy :: Proxy b)
140+ (oldf, old) = genRequest (Proxy :: Proxy b)
117141 param = cs $ symbolVal (Proxy :: Proxy x)
118142 new = arbitrary :: Gen [c]
119143 toParam c = param <> "[]=" <> cs (toQueryParam c)
@@ -121,23 +145,23 @@ instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
121145
122146instance (KnownSymbol x, HasGenRequest b)
123147 => HasGenRequest (QueryFlag x :> b) where
124− genRequest _ = do
148+ genRequest _ = (oldf, do
125149 old' <- old
126150 return $ \burl -> let r = old' burl
127151 qs = queryString r in r {
128− queryString = if BS.null qs then param else param <> "&" <> qs }
152+ queryString = if BS.null qs then param else param <> "&" <> qs })
129153 where
130− old = genRequest (Proxy :: Proxy b)
154+ (oldf, old) = genRequest (Proxy :: Proxy b)
131155 param = cs $ symbolVal (Proxy :: Proxy x)
132156
133157instance (ReflectMethod method)
134158 => HasGenRequest (Verb (method :: k) (status :: Nat) (cts :: [*]) a) where
135− genRequest _ = return $ \burl -> defaultRequest
159+ genRequest _ = (1, return $ \burl -> defaultRequest
136160 { host = cs $ baseUrlHost burl
137161 , port = baseUrlPort burl
138162 , secure = baseUrlScheme burl == Https
139163 , method = reflectMethod (Proxy :: Proxy method)
140− }
164+ })
141165
142166instance (HasGenRequest a) => HasGenRequest (RemoteHost :> a) where
143167 genRequest _ = genRequest (Proxy :: Proxy a)
src/Servant/QuickCheck/Internal/QuickCheck.hs+3−3
@@ -71,7 +71,7 @@ withServantServerAndContext api ctx server t
7171serversEqual :: HasGenRequest a =>
7272 Proxy a -> BaseUrl -> BaseUrl -> Args -> ResponseEquality LBS.ByteString -> Expectation
7373serversEqual api burl1 burl2 args req = do
74− let reqs = (\f -> (f burl1, f burl2)) <$> genRequest api
74+ let reqs = (\f -> (f burl1, f burl2)) <$> runGenRequest api
7575 -- This MVar stuff is clunky! But there doesn't seem to be an easy way to
7676 -- return results when a test fails, since an exception is throw.
7777 deetsMVar <- newMVar $ error "should not be called"
@@ -113,7 +113,7 @@ serversEqual api burl1 burl2 args req = do
113113serverSatisfies :: (HasGenRequest a) =>
114114 Proxy a -> BaseUrl -> Args -> Predicates -> Expectation
115115serverSatisfies api burl args preds = do
116− let reqs = ($ burl) <$> genRequest api
116+ let reqs = ($ burl) <$> runGenRequest api
117117 deetsMVar <- newMVar $ error "should not be called"
118118 r <- quickCheckWithResult args { chatty = False } $ monadicIO $ forAllM reqs $ \req -> do
119119 v <- run $ finishPredicates preds (noCheckStatus req) defManager
@@ -133,7 +133,7 @@ serverSatisfies api burl args preds = do
133133serverDoesntSatisfy :: (HasGenRequest a) =>
134134 Proxy a -> BaseUrl -> Args -> Predicates -> Expectation
135135serverDoesntSatisfy api burl args preds = do
136− let reqs = ($ burl) <$> genRequest api
136+ let reqs = ($ burl) <$> runGenRequest api
137137 r <- quickCheckWithResult args $ monadicIO $ forAllM reqs $ \req -> do
138138 v <- run $ finishPredicates preds (noCheckStatus req) defManager
139139 assert $ not $ null v
test/Servant/QuickCheck/InternalSpec.hs+67−19
@@ -1,19 +1,21 @@
11{-# LANGUAGE CPP #-}
22module Servant.QuickCheck.InternalSpec (spec) where
33
4−import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
5−import Control.Monad.IO.Class (liftIO)
6−import qualified Data.ByteString as BS
7−import qualified Data.ByteString.Char8 as C
4+import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
5+import Control.Monad (replicateM)
6+import Control.Monad.IO.Class (liftIO)
7+import qualified Data.ByteString as BS
8+import qualified Data.ByteString.Char8 as C
9+import Data.Maybe (fromJust)
810import Prelude.Compat
911import Servant
10−import Test.Hspec (Spec, context, describe, it, shouldBe,
11− shouldContain)
12−import Test.Hspec.Core.Spec (Arg, Example, Result (..),
13− defaultParams, evaluateExample)
14−import Test.QuickCheck.Gen (unGen)
15−import Test.QuickCheck.Random (mkQCGen)
16−import Network.HTTP.Client (queryString, path)
12+import Test.Hspec (Spec, context, describe, it, shouldBe,
13+ shouldContain)
14+import Test.Hspec.Core.Spec (Arg, Example, Result (..),
15+ defaultParams, evaluateExample)
16+import Test.QuickCheck.Gen (unGen, generate)
17+import Test.QuickCheck.Random (mkQCGen)
18+import Network.HTTP.Client (queryString, path)
1719
1820#if MIN_VERSION_servant(0,8,0)
1921import Servant.API.Internal.Test.ComprehensiveAPI (comprehensiveAPIWithoutRaw)
@@ -23,7 +25,8 @@ import Servant.API.Internal.Test.ComprehensiveAPI (ComprehensiveAPI,
2325#endif
2426
2527import Servant.QuickCheck
26−import Servant.QuickCheck.Internal (genRequest, serverDoesntSatisfy)
28+import Servant.QuickCheck.Internal (genRequest, runGenRequest, serverDoesntSatisfy)
29+
2730
2831spec :: Spec
2932spec = do
@@ -35,6 +38,7 @@ spec = do
3538 queryParamsSpec
3639 queryFlagsSpec
3740 deepPathSpec
41+ unbiasedGenerationSpec
3842
3943serversEqualSpec :: Spec
4044serversEqualSpec = describe "serversEqual" $ do
@@ -45,8 +49,6 @@ serversEqualSpec = describe "serversEqual" $ do
4549 serversEqual api burl1 burl2 args bodyEquality
4650
4751 context "when servers are not equal" $ do
48−
49−
5052 it "provides the failing responses in the error message" $ do
5153 Fail _ err <- withServantServer api2 server2 $ \burl1 ->
5254 withServantServer api2 server3 $ \burl2 -> do
@@ -76,11 +78,12 @@ serverSatisfiesSpec = describe "serverSatisfies" $ do
7678
7779 it "fails with informative error messages" $ do
7880 Fail _ err <- withServantServerAndContext api ctx server $ \burl -> do
79− evalExample $ serverSatisfies api burl args (getsHaveCacheControlHeader <%> mempty)
80− err `shouldContain` "getsHaveCacheControlHeader"
81+ evalExample $ serverSatisfies api burl args (notAllowedContainsAllowHeader <%> mempty)
82+ err `shouldContain` "notAllowedContainsAllowHeader"
8183 err `shouldContain` "Headers"
8284 err `shouldContain` "Body"
8385
86+
8487onlyJsonObjectSpec :: Spec
8588onlyJsonObjectSpec = describe "onlyJsonObjects" $ do
8689
@@ -120,7 +123,7 @@ deepPathSpec = describe "Path components" $ do
120123 it "are separated by slashes, without a trailing slash" $ do
121124 let rng = mkQCGen 0
122125 burl = BaseUrl Http "localhost" 80 ""
123− gen = genRequest deepAPI
126+ gen = runGenRequest deepAPI
124127 req = (unGen gen rng 0) burl
125128 path req `shouldBe` ("/one/two/three")
126129
@@ -131,7 +134,7 @@ queryParamsSpec = describe "QueryParams" $ do
131134 it "reduce to an HTTP query string correctly" $ do
132135 let rng = mkQCGen 0
133136 burl = BaseUrl Http "localhost" 80 ""
134− gen = genRequest paramsAPI
137+ gen = runGenRequest paramsAPI
135138 req = (unGen gen rng 0) burl
136139 qs = C.unpack $ queryString req
137140 qs `shouldBe` "one=_&two=_"
@@ -142,11 +145,33 @@ queryFlagsSpec = describe "QueryFlags" $ do
142145 it "reduce to an HTTP query string correctly" $ do
143146 let rng = mkQCGen 0
144147 burl = BaseUrl Http "localhost" 80 ""
145− gen = genRequest flagsAPI
148+ gen = runGenRequest flagsAPI
146149 req = (unGen gen rng 0) burl
147150 qs = C.unpack $ queryString req
148151 qs `shouldBe` "one&two"
149152
153+makeRandomRequest :: Proxy LargeAPI -> BaseUrl -> IO Integer
154+makeRandomRequest large burl = do
155+ req <- generate $ runGenRequest large
156+ pure $ fst . fromJust . C.readInteger . C.drop 1 . path $ req burl
157+
158+
159+unbiasedGenerationSpec :: Spec
160+unbiasedGenerationSpec = describe "Unbiased Generation of requests" $
161+
162+ it "frequency paired with generated endpoint should be more randomly distributed" $ do
163+ let burl = BaseUrl Http "localhost" 80 ""
164+ let runs = 10000 :: Double
165+ someRequests <- replicateM 10000 (makeRandomRequest largeApi burl)
166+ let mean = (sum $ map fromIntegral someRequests) / runs
167+ let variancer x = let ix = fromIntegral x in (ix - mean) * (ix - mean)
168+ let variance = (sum $ map variancer someRequests) / runs - 1
169+ -- mean should be around 8
170+ mean > 7 `shouldBe` True
171+ mean < 9 `shouldBe` True
172+ -- Std dev is likely around 4. Variance is probably greater than 20.
173+ variance > 19.5 `shouldBe` True
174+
150175------------------------------------------------------------------------------
151176-- APIs
152177------------------------------------------------------------------------------
@@ -194,6 +219,29 @@ server2 = return $ return 1
194219server3 :: IO (Server API2)
195220server3 = return $ return 2
196221
222+
223+largeApi :: Proxy LargeAPI
224+largeApi = Proxy
225+
226+type LargeAPI
227+ = "1" :> Get '[JSON] Int
228+ :<|> "2" :> Get '[JSON] Int
229+ :<|> "3" :> Get '[JSON] Int
230+ :<|> "4" :> Get '[JSON] Int
231+ :<|> "5" :> Get '[JSON] Int
232+ :<|> "6" :> Get '[JSON] Int
233+ :<|> "7" :> Get '[JSON] Int
234+ :<|> "8" :> Get '[JSON] Int
235+ :<|> "9" :> Get '[JSON] Int
236+ :<|> "10" :> Get '[JSON] Int
237+ :<|> "11" :> Get '[JSON] Int
238+ :<|> "12" :> Get '[JSON] Int
239+ :<|> "13" :> Get '[JSON] Int
240+ :<|> "14" :> Get '[JSON] Int
241+ :<|> "15" :> Get '[JSON] Int
242+ :<|> "16" :> Get '[JSON] Int
243+
244+
197245type OctetAPI = Get '[OctetStream] BS.ByteString
198246
199247octetAPI :: Proxy OctetAPI