git.iko.soy/servant-quickcheck
on master help sign in
Rewrite.
2050487Julian K. Arni
servant-quickcheck.cabal+8−1
@@ -22,7 +22,7 @@ library
22 , Servant.QuickCheck.Internal
22 , Servant.QuickCheck.Internal
23 , Servant.QuickCheck.Internal.Benchmarking
23 , Servant.QuickCheck.Internal.Benchmarking
24 , Servant.QuickCheck.Internal.Predicates
24 , Servant.QuickCheck.Internal.Predicates
25− , Servant.QuickCheck.Internal.Testable
25+ , Servant.QuickCheck.Internal.HasGenRequest
26 , Servant.QuickCheck.Internal.QuickCheck
26 , Servant.QuickCheck.Internal.QuickCheck
27 build-depends: base >=4.8 && <4.9
27 build-depends: base >=4.8 && <4.9
28 , QuickCheck == 2.8.*
28 , QuickCheck == 2.8.*
@@ -31,8 +31,11 @@ library
31 , mtl == 2.2.*
31 , mtl == 2.2.*
32 , http-client == 0.4.*
32 , http-client == 0.4.*
33 , http-types == 0.9.*
33 , http-types == 0.9.*
34+ , http-media
34 , servant-client == 0.7.*
35 , servant-client == 0.7.*
35 , servant-server == 0.7.*
36 , servant-server == 0.7.*
37+ , string-conversions == 0.4.*
38+ , data-default-class == 0.0.*
36 , servant == 0.7.*
39 , servant == 0.7.*
37 , warp >= 3.2.4 && < 3.3
40 , warp >= 3.2.4 && < 3.3
38 , process == 1.2.*
41 , process == 1.2.*
@@ -46,9 +49,12 @@ library
46 , GADTs
49 , GADTs
47 , MultiParamTypeClasses
50 , MultiParamTypeClasses
48 , DeriveFunctor
51 , DeriveFunctor
52+ , KindSignatures
49 , RankNTypes
53 , RankNTypes
50 , ConstraintKinds
54 , ConstraintKinds
51 , DeriveGeneric
55 , DeriveGeneric
56+ , ScopedTypeVariables
57+ , OverloadedStrings
52 default-language: Haskell2010
58 default-language: Haskell2010
53
59
54test-suite spec
60test-suite spec
@@ -67,6 +73,7 @@ test-suite spec
67 , servant-client
73 , servant-client
68 , transformers
74 , transformers
69 , QuickCheck
75 , QuickCheck
76+ , quickcheck-io
70 default-extensions: TypeOperators
77 default-extensions: TypeOperators
71 , FlexibleInstances
78 , FlexibleInstances
72 , FlexibleContexts
79 , FlexibleContexts
src/Servant/QuickCheck.hs+1−66
@@ -19,76 +19,11 @@
19module Servant.QuickCheck
19module Servant.QuickCheck
20 (
20 (
21
21
22− -- * Server properties
23− -- | Functions to verify that a server meets certain properties.
24− --
25− -- Example:
26− --
27− -- > server :: Server API
28− -- > server = return . show
29− -- >
30− -- >
31− -- > test :: Spec
32− -- > test = describe "my server" $ do
33− -- >
34− -- > it "never throws a 500 on valid input" $ do
35− -- > withServantServer api server $ \url ->
36− -- > serverSatisfiers api url emptyPredicates never500s 100
37− serverSatisfies
38−
39− -- * Server equality
40− -- | Functions to verify that two servers behave identically.
41− --
42− -- This can be useful when for example rewriting or refactoring an
43− -- application.
44− --
45− -- Example:
46− --
47− -- > server :: Server API
48− -- > server = return . show
49− -- >
50− -- > server2 :: Server API
51− -- > server2 = const $ return "hi"
52− -- >
53− -- > test :: Spec
54− -- > test = describe "my new server" $ do
55− -- >
56− -- > it "behaves like the old one" $ do
57− -- > withServantServer api server $ \url1 ->
58− -- > withServantServer api server2 $ \url2 ->
59− -- > serversEqual api url1 url2 100
60− --
61− , serversEqual
22+ serversEqual
62−
63− -- * Server benchmarking
64− -- | Functions that randomly generate and run benchmarking scripts
65− , serverBenchmark
66− , BenchOptions(..)
67− , defaultBenchOptions
68−
69−
70 -- * Test setup helpers
23 -- * Test setup helpers
71 -- | Helpers to setup and teardown @servant@ servers during tests.
24 -- | Helpers to setup and teardown @servant@ servers during tests.
72 , withServantServer
25 , withServantServer
73
26
74− -- * Predicates
75− -- | Predicates (functions with signatures @a -> Bool@) are used to filter
76− -- out QuickCheck-generated values (so as to specify that requests must
77− -- possess certain properties) and to check that the response specifies the
78− -- expected properties.
79− , Predicates
80− , emptyPredicates
81− , addPredicate
82− , addPolyPredicate
83−
84− -- ** Predicate convenience functions
85− , addRightPredicate
86− , addLeftPredicate
87−
88− -- ** Useful predicates
89− , never500s
90− , onlyJsonObjects
91−
92 -- ** Re-exports
27 -- ** Re-exports
93 , BaseUrl(..)
28 , BaseUrl(..)
94 , Scheme(..)
29 , Scheme(..)
src/Servant/QuickCheck/Internal.hs+1−1
@@ -1,6 +1,6 @@
1module Servant.QuickCheck.Internal (module X) where
1module Servant.QuickCheck.Internal (module X) where
2
2
3−import Servant.QuickCheck.Internal.Testable as X
3+import Servant.QuickCheck.Internal.HasGenRequest as X
4import Servant.QuickCheck.Internal.Predicates as X
4import Servant.QuickCheck.Internal.Predicates as X
5import Servant.QuickCheck.Internal.QuickCheck as X
5import Servant.QuickCheck.Internal.QuickCheck as X
6import Servant.QuickCheck.Internal.Benchmarking as X
6import Servant.QuickCheck.Internal.Benchmarking as X
src/Servant/QuickCheck/Internal/HasGenRequest.hsnew file+93
@@ -0,0 +1,93 @@
1+{-# LANGUAGE PolyKinds #-}
2+module Servant.QuickCheck.Internal.HasGenRequest where
3+
4+import Data.Default.Class (def)
5+import Data.Monoid ((<>))
6+import Data.String (fromString)
7+import Data.String.Conversions (cs)
8+import GHC.TypeLits
9+import Network.HTTP.Client (Request, RequestBody (..), host,
10+ method, path, port, requestBody,
11+ requestHeaders, secure, queryString)
12+import Network.HTTP.Media (renderHeader)
13+import Servant
14+import Servant.API.ContentTypes
15+import Servant.Client (BaseUrl (..), Scheme (..))
16+import Test.QuickCheck
17+
18+
19+class HasGenRequest a where
20+ genRequest :: Proxy a -> Gen (BaseUrl -> Request)
21+
22+instance (HasGenRequest a, HasGenRequest b) => HasGenRequest (a :<|> b) where
23+ genRequest _
24+ = oneof [ genRequest (Proxy :: Proxy a)
25+ , genRequest (Proxy :: Proxy b)
26+ ]
27+
28+instance (KnownSymbol path, HasGenRequest b) => HasGenRequest (path :> b) where
29+ genRequest _ = do
30+ old' <- old
31+ return $ \burl -> let r = old' burl in r { path = new <> "/" <> path r }
32+ where
33+ old = genRequest (Proxy :: Proxy b)
34+ new = cs $ symbolVal (Proxy :: Proxy path)
35+
36+instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
37+ => HasGenRequest (Capture x c :> b) where
38+ genRequest _ = do
39+ old' <- old
40+ new' <- toUrlPiece <$> new
41+ return $ \burl -> let r = old' burl in r { path = cs new' <> "/" <> path r }
42+ where
43+ old = genRequest (Proxy :: Proxy b)
44+ new = arbitrary :: Gen c
45+
46+instance (Arbitrary c, KnownSymbol h, HasGenRequest b, ToHttpApiData c)
47+ => HasGenRequest (Header h c :> b) where
48+ genRequest _ = do
49+ old' <- old
50+ new' <- toUrlPiece <$> new
51+ return $ \burl -> let r = old' burl in r {
52+ requestHeaders = (hdr, cs new') : requestHeaders r }
53+ where
54+ old = genRequest (Proxy :: Proxy b)
55+ hdr = fromString $ symbolVal (Proxy :: Proxy h)
56+ new = arbitrary :: Gen c
57+
58+instance (AllMimeRender x c, Arbitrary c, HasGenRequest b)
59+ => HasGenRequest (ReqBody x c :> b) where
60+ genRequest _ = do
61+ old' <- old
62+ new' <- new
63+ (ct, bd) <- elements $ allMimeRender (Proxy :: Proxy x) new'
64+ return $ \burl -> let r = old' burl in r {
65+ requestBody = RequestBodyLBS bd
66+ , requestHeaders = ("Content-Type", renderHeader ct) : requestHeaders r
67+ }
68+ where
69+ old = genRequest (Proxy :: Proxy b)
70+ new = arbitrary :: Gen c
71+
72+instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
73+ => HasGenRequest (QueryParam x c :> b) where
74+ genRequest _ = do
75+ new' <- new
76+ old' <- old
77+ return $ \burl -> let r = old' burl in r {
78+ queryString = queryString r
79+ <> param <> "=" <> cs (toQueryParam new') }
80+ where
81+ old = genRequest (Proxy :: Proxy b)
82+ param = cs $ symbolVal (Proxy :: Proxy x)
83+ new = arbitrary :: Gen c
84+
85+instance (ReflectMethod method)
86+ => HasGenRequest (Verb (method :: k) (status :: Nat) (cts :: [*]) a) where
87+ genRequest _ = return $ \burl -> def
88+ { host = cs $ baseUrlHost burl
89+ , port = baseUrlPort burl
90+ , secure = baseUrlScheme burl == Https
91+ , method = reflectMethod (Proxy :: Proxy method)
92+ }
93+
src/Servant/QuickCheck/Internal/Predicates.hs+142−125
@@ -1,132 +1,149 @@
1−-- | This module contains all logic related to constructing or using
2−-- @Predicates@.
3−{-# LANGUAGE UndecidableInstances #-}
4−{-# LANGUAGE ScopedTypeVariables #-}
5module Servant.QuickCheck.Internal.Predicates where
1module Servant.QuickCheck.Internal.Predicates where
6
2
7−import Data.Aeson (ToJSON (toJSON), Value (..))
3+import Data.Monoid ((<>))
8−import Data.Proxy (Proxy (..))
4+import GHC.Generics (Generic)
9−import Data.Void
5+import Control.Monad
10−import Network.HTTP.Types (statusCode)
6+import Network.HTTP.Client (Request, Response)
11−import Servant.Common.Req (ServantError (..))
12−import Test.QuickCheck
13
7
14−
8+{-
15−-- | An HList containing predicates (functions of type @a -> Bool@). This
9+-- | @500 Internal Server Error@ should be avoided - it may represent some
16−-- datatype is used to represent both filters (what values to discard when
10+-- issue with the application code, and it moreover gives the client little
17−-- generating arguments to test an API) and tests results (what to consider a
11+-- indication of how to proceed or what went wrong.
18−-- failing response).
19−--
20−-- For both filters and test results, only the *first* predicate of the
21−-- appropriate type is used.
22−--
23−-- Use 'emptyPredicates', 'addPredicate', 'addLeftPredicate' and
24−-- 'addRightPredicate' to construct a @Predicates@.
25−data Predicates a where
26− HNil :: Predicates '[]
27− HCons :: (a -> Bool) -> Predicates b -> Predicates (a ': b)
28− HConsC :: Constraint a -> Predicates b -> Predicates (Constraint a ': b)
29−
30−class HasPredicate a b where
31− getPredicate :: Predicates a -> b -> Bool
32−
33−instance {-# OVERLAPPING #-} HasPredicate '[] a where
34− getPredicate _ = const True
35−
36−-- TODO: Find some better way of distinguishing how the predicate is being used
37−instance {-# OVERLAPPING #-} HasPredicate '[] (Either ServantError a) where
38− getPredicate _ = discard
39−
40−instance {-# OVERLAPPING #-} HasPredicate (a ': xs) a where
41− getPredicate (HCons a _) = a
42− getPredicate (HConsC _ _) = error "not impossible, but non-sensical"
43−
44−data Constraint ctx = Constraint
45− { getConstraint :: forall a . (ctx a) => a -> Bool }
46−
47−-- This is a little bit of a hack. Ideally instances would match when the
48−-- predicate is polymorphic, but that doesn't work since the polymorphic type
49−-- may have to unify with multiple distict values.
50−--
51−-- It may however be possible to define a MPTC from monomorphic to polymorphic
52−-- datatypes to avoid this issue.
53−instance {-# OVERLAPPING #-}
54− HasPredicate (Either ServantError Void ': xs) (Either ServantError a) where
55− getPredicate (HCons f _) x = case x of
56− Left e -> f (Left e)
57− Right _ -> True
58−
59−instance {-# OVERLAPPING #-} (ctx a)
60− => HasPredicate (Constraint ctx ': xs) (Either ServantError a) where
61− getPredicate (HConsC f _) x = case x of
62− Left _ -> discard -- Not clear whether checking for FailureResponse is better
63− Right v -> getConstraint f v
64− getPredicate (HCons _ _) _ = error "not impossible, but non-sensical"
65−
66−instance {-# OVERLAPPABLE #-} (ls ~ (b ': xs), HasPredicate xs a)
67− => HasPredicate ls a where
68− getPredicate (HCons _ xs) = getPredicate xs
69− getPredicate _ = error "impossible"
70−
71−-- | Add a predicate to a list of predicates. Note that the predicate may not
72−-- be polymorphic.
73−addPredicate :: (a -> Bool) -> Predicates b -> Predicates (a ': b)
74−addPredicate = HCons
75−
76−-- | Add a predicate with a class constraint.
77−--
78−-- Note that every possible argument must be an instance of that class for this
79−-- to typecheck. In other words, if the @Predicates@ is being used for return
80−-- types, every return type in the API must be an instance of the class. If
81−-- it's being used for filtering, every capture, header, body, etc. type must
82−-- be an instance of that class.
83−--
84−-- This can be used to for example test that returned JSON has certain
85−-- properties, or (via generics) that if any datatype contains a (possibly
86−-- nested) field of a particular type, it always meets certain properties.
87−addPolyPredicate :: proxy ctx -> (forall a. ctx a => a -> Bool) -> Predicates b
88− -> Predicates (Constraint ctx ': b)
89−addPolyPredicate _ p = HConsC (Constraint p)
90−
91−-- | Given a predicate over an @p :: a -> Bool@, add a predicate to the @Predicates@
92−-- list that succeeds on an @val :: Either ServantError a@ if @val@ is a
93−-- @Left@, or a @Right v@ such that @p a == True@.
94−addRightPredicate :: (a -> Bool) -> Predicates b -> Predicates (Either ServantError a ': b)
95−addRightPredicate p = addPredicate $ either (const True) p
96−
97−-- | The @Left@ analog of 'addRightPredicate'.
98−addLeftPredicate :: (ServantError -> Bool) -> Predicates b
99− -> Predicates (Either ServantError Void ': b)
100−addLeftPredicate p = addPredicate $ either p (error "impossible")
101−
102−-- | An empty list of predicates. This doesn't discard any values when used as
103−-- a filter, and doesn't fail any value when used as a condition to satisfy.
104−emptyPredicates :: Predicates '[]
105−emptyPredicates = HNil
106−
107−-- * Useful predicates
108−
109−-- | A @Predicates@ list that fails a test if the response is an HTTP 500 error.
110−never500s :: Predicates '[Either ServantError Void]
111−never500s = addLeftPredicate go emptyPredicates
112− where
113− go (FailureResponse x _ _) = statusCode x /= 500
114− go _ = True
115−
116−-- | A @Predicates@ list that fails a test if the response is anything but a
117−-- top-level object (e.g., if it is an array or literal).
118--
12--
13+-- This function checks that the response code is not 500.
14+not500 :: Response b -> IO Bool
15+not500
16+ = ResponsePredicate "not500" _
17+
119−-- Returning anything other than object is considered bad practice, as
18+-- | Returning anything other than an object when returning JSON is considered
19+-- bad practice, as:
120--
20--
121-- (1) it is hard to modify the returned value while maintaining backwards
21-- (1) it is hard to modify the returned value while maintaining backwards
122−-- compatibility;
22+-- compatibility
123−-- (2) many older tools do not support top-level arrays;
23+-- (2) many older tools do not support top-level arrays
124-- (3) whether top-level numbers, booleans, or strings are valid JSON depends
24-- (3) whether top-level numbers, booleans, or strings are valid JSON depends
125−-- on what RFC you're going by;
25+-- on what RFC you're going by
126−-- (4) there are security issues with top-level arrays.
26+-- (4) there are security issues with top-level arrays
27+--
28+-- This function checks that any @application/json@ responses only return JSON
29+-- objects (and not arrays, strings, numbers, or booleans) at the top level.
127−onlyJsonObjects :: Predicates '[Constraint ToJSON]
30+onlyJsonObjects :: Response b -> IO Bool
128−onlyJsonObjects = addPolyPredicate (Proxy :: Proxy ToJSON) go emptyPredicates
31+onlyJsonObjects
129− where
32+ = ResponsePredicate "onlyJsonObjects" _
130− go x = case toJSON x of
33+
131− Object _ -> True
34+-- | When creating a new resource, it is good practice to provide a @Location@
132− _ -> False
35+-- header with a link to the created resource.
36+--
37+-- This function checks that every @201 Created@ response contains a @Location@
38+-- header, and that the link in it responds with a 2XX response code to @GET@
39+-- requests.
40+--
41+-- References: <RFC 7231, Section 6.3.2 https://tools.ietf.org/html/rfc7231#section-6.3.2>
42+createContainsValidLocation :: Response b -> IO Bool
43+createContainsValidLocation
44+ = ResponsePredicate "createContainsValidLocation" _
45+
46+getsHaveLastModifiedHeader :: Response b -> IO Bool
47+getsHaveLastModifiedHeader
48+ = ResponsePredicate "getsHaveLastModifiedHeader" _
49+
50+-- | When an HTTP request has a method that is not allowed, a 405 response
51+-- should be returned. Additionally, it is good practice to return an @Allow@
52+-- header with the list of allowed methods.
53+--
54+-- This function checks that every @405 Method Not Allowed@ response contains
55+-- an @Allow@ header with a list of standard HTTP methods.
56+notAllowedContainsAllowHeader :: Response b -> IO Bool
57+notAllowedContainsAllowHeader
58+ = ResponsePredicate "notAllowedContainsAllowHeader" _
59+
60+-- | When a request contains an @Accept@ header, the server must either return
61+-- content in one of the requested representations, or respond with @406 Not
62+-- Acceptable@.
63+--
64+-- This function checks that every *successful* response has a @Content-Type@
65+-- header that matches the @Accept@ header.
66+honoursAcceptHeader :: Predicate b Bool
67+honoursAcceptHeader
68+ = RequestPredicate "honoursAcceptHeader" _
69+
70+-- | Whether or not a representation should be cached, it is good practice to
71+-- have a @Cache-Control@ header for @GET@ requests. If the representation
72+-- should not be cached, used @Cache-Control: no-cache@.
73+--
74+-- This function checks that @GET@ responses have a valid @Cache-Control@
75+-- header.
76+--
77+-- References: RFC 7234 Section 5.2
78+-- https://tools.ietf.org/html/rfc7234#section-5.2
79+getsHaveCacheControlHeader :: Predicate b Bool
80+getsHaveCacheControlHeader
81+ = ResponsePredicate "getsHaveCacheControlHeader" _
82+
83+-- | Like 'getsHaveCacheControlHeader', but for @HEAD@ requests.
84+headsHaveCacheControlHeader :: Predicate b Bool
85+headsHaveCacheControlHeader
86+ = ResponsePredicate "headsHaveCacheControlHeader" _
87+
88+-- |
89+--
90+-- If the original request modifies the resource, this function makes two
91+-- requests:
92+--
93+-- (1) Once, with the original request and a future date as the
94+-- @If-Unmodified-Since@, which is expected to succeed.
95+-- (2) Then with the original request again, with a @If-Unmodified-Since@
96+-- safely in the past. Since presumably the representation has been changed
97+-- recently (by the first request), this is expected to fail with @412
98+-- Precondition Failure@.
99+--
100+-- Note that the heuristic used to guess whether the original request modifies
101+-- a resource is simply whether the method is @PUT@ or @PATCH@, which may be
102+-- incorrect in certain circumstances.
103+supportsIfUnmodifiedSince :: Predicate b Bool
104+supportsIfUnmodifiedSince
105+ = ResponsePredicate "supportsIfUnmodifiedSince" _
106+
107+-- | @OPTIONS@ responses should contain an @Allow@ header with the list of
108+-- allowed methods.
109+--
110+-- If a request is an @OPTIONS@ request, and if the response is a successful
111+-- one, this function checks the response for an @Allow@ header. It fails if:
112+--
113+-- (1) There is no @Allow@ header
114+-- (2) The @Allow@ header does not have standard HTTP methods in the correct
115+-- format
116+-- (3) Making a request to the same URL with one of those methods results in
117+-- a 404 or 405.
118+optionsContainsValidAllow :: Predicate b Bool
119+optionsContainsValidAllow
120+ = ResponsePredicate "optionsContainsValidAllow" _
121+
122+-- | Link headers are a standardized way of presenting links that may be
123+-- relevant to a client.
124+--
125+-- This function checks that any @Link@ headers have values in the correct
126+-- format.
127+--
128+-- References: RFC 5988 Section 5
129+-- https://tools.ietf.org/html/rfc5988
130+linkHeadersAreValid :: Predicate b Bool
131+linkHeadersAreValid
132+ = ResponsePredicate "linkHeadersAreValid" _
133+
134+-- | Any @401 Unauthorized@ response must include a @WWW-Authenticate@ header.
135+--
136+-- This function checks that, if a response has status code 401, it contains a
137+-- @WWW-Authenticate@ header.
138+--
139+-- References: RFC 7235 Section 4.1
140+-- https://tools.ietf.org/html/rfc7235#section-4.1
141+unauthorizedContainsWWWAuthenticate :: Predicate b Bool
142+unauthorizedContainsWWWAuthenticate
143+ = ResponsePredicate "unauthorizedContainsWWWAuthenticate" _
144+-}
145+
146+data Predicate b r
147+ = ResponsePredicate String (Response b -> IO r)
148+ | RequestPredicate String (Request -> [Response b -> IO r] -> IO r)
149+ deriving (Generic)
src/Servant/QuickCheck/Internal/QuickCheck.hs+13−123
@@ -7,7 +7,7 @@ import Control.Monad (replicateM_)
7import Data.Proxy (Proxy)
7import Data.Proxy (Proxy)
8import Data.Void (Void)
8import Data.Void (Void)
9import Network.HTTP.Client (Manager, defaultManagerSettings,
9import Network.HTTP.Client (Manager, defaultManagerSettings,
10− newManager)
10+ newManager, httpLbs)
11import Network.HTTP.Client (managerModifyRequest, getUri)
11import Network.HTTP.Client (managerModifyRequest, getUri)
12import Network.Wai.Handler.Warp (withApplication)
12import Network.Wai.Handler.Warp (withApplication)
13import Servant (HasServer, Server, serve)
13import Servant (HasServer, Server, serve)
@@ -18,11 +18,11 @@ import System.IO.Temp (withSystemTempFile)
18import System.Mem (performGC)
18import System.Mem (performGC)
19import System.Process (callCommand)
19import System.Process (callCommand)
20import Test.Hspec (Expectation, expectationFailure)
20import Test.Hspec (Expectation, expectationFailure)
21−import Test.QuickCheck (Args (..), Property, Result (..),
21+import Test.QuickCheck (Args (..), Property, forAll, Result (..),
22− Testable, property,
22+ Testable, property, ioProperty,
23 quickCheckWithResult, stdArgs)
23 quickCheckWithResult, stdArgs)
24
24
25−import Servant.QuickCheck.Internal.Testable
25+import Servant.QuickCheck.Internal.HasGenRequest
26import Servant.QuickCheck.Internal.Predicates
26import Servant.QuickCheck.Internal.Predicates
27import Servant.QuickCheck.Internal.Benchmarking
27import Servant.QuickCheck.Internal.Benchmarking
28
28
@@ -35,27 +35,6 @@ withServantServer api server t
35 = withApplication (return . serve api =<< server) $ \port ->
35 = withApplication (return . serve api =<< server) $ \port ->
36 t (BaseUrl Http "localhost" port "")
36 t (BaseUrl Http "localhost" port "")
37
37
38−-- | A QuickCheck 'Property' that randomly generates arguments (captures, query
39−-- params, request bodies, headers, etc.) expected by endpoints of a server,
40−-- and makes requests to the servers running in the two provided URLs in the
41−-- same order, failing if they do not return the same response.
42−--
43−-- Evidently, if the behaviour of the server is expected to be
44−-- non-deterministic, this function may produce spurious failures.
45−--
46−-- Note that this QuickCheck 'Property' does IO; interleaving it with other IO
47−-- actions will not work. It is provided so that it can be used with QuickCheck
48−-- functions such as 'quickCheckWith'. For most use cases, you should use
49−-- @serversEqual@ or @servantServersEqual@.
50−serversEqualProperty :: (HasClient a, Testable (ShouldMatch (Client a)))
51− => Proxy a -> Manager -> BaseUrl -> BaseUrl -> Property
52−serversEqualProperty api mgr burl1 burl2
53− = property $ ShouldMatch
54− { smClient = client api
55− , smManager = mgr
56− , smBaseUrls = (burl1, burl2)
57− }
58−
59-- | Check that the two servers running under the provided @BaseUrl@s behave
38-- | Check that the two servers running under the provided @BaseUrl@s behave
60-- identically by randomly generating arguments (captures, query params, request bodies,
39-- identically by randomly generating arguments (captures, query params, request bodies,
61-- headers, etc.) expected by the server. If, given the same request, the
40-- headers, etc.) expected by the server. If, given the same request, the
@@ -66,103 +45,14 @@ serversEqualProperty api mgr burl1 burl2
66-- run.
45-- run.
67--
46--
68-- Evidently, if the behaviour of the server is expected to be
47-- Evidently, if the behaviour of the server is expected to be
69−-- non-deterministic, this function may produce spurious failures.
48+-- non-deterministic, this function may produce spurious failures
70−serversEqual :: (HasClient a, Testable (ShouldMatch (Client a)))
49+serversEqual :: HasGenRequest a =>
71− => Proxy a -> BaseUrl -> BaseUrl -> Int -> Expectation
50+ Proxy a -> BaseUrl -> BaseUrl -> Manager -> Property
72−serversEqual api burl1 burl2 tries = do
51+serversEqual api burl1 burl2 mgr =
73− mgr <- managerWithStoredReq
52+ let reqs = (\f -> (f burl1, f burl2)) <$> genRequest api
74− let args = stdArgs { chatty = False, maxSuccess = tries }
53+ in forAll reqs $ \(req1, req2) -> ioProperty $ do
75− res <- quickCheckWithResult args $ serversEqualProperty api mgr burl1 burl2
54+ resp1 <- httpLbs req1 mgr
76− case res of
55+ resp2 <- httpLbs req2 mgr
77− Success _ _ _ -> return ()
56+ return $ resp1 == resp2
78− _ -> prettyErr >>= expectationFailure
79−
80−
81−serverSatisfiesProperty :: (HasClient a, Testable (ShouldSatisfy filt exp (Client a)))
82− => Proxy a -> Manager -> BaseUrl -> Predicates filt -> Predicates exp -> Property
83−serverSatisfiesProperty api mgr burl filters expect = do
84− property $ ShouldSatisfy
85− { ssVal = client api
86− , ssFilter = filters
87− , ssExpect = expect
88− , ssManager = mgr
89− , ssBaseUrl = burl
90− }
91
57
92−-- | Check that a server's responses satisfies certain properties.
93−serverSatisfies :: (HasClient a, Testable (ShouldSatisfy filt exp (Client a)))
94− => Proxy a -> BaseUrl -> Predicates filt -> Predicates exp
95− -> Int -> Expectation
96−serverSatisfies api burl filters expect tries = do
97− mgr <- managerWithStoredReq
98− let args = stdArgs { chatty = False, maxSuccess = tries }
99− res <- quickCheckWithResult args $ serverSatisfiesProperty api mgr burl filters expect
100− case res of
101− Success _ _ _ -> return ()
102− GaveUp n _ _ -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
103− _ -> prettyErr >>= expectationFailure
104−
105−-- | Check that the two servers running under the provided @BaseUrl@s do not
106−-- behave identically.
107−--
108−-- As with @serversEqualProperty@, non-determinism in the servers will likely
109−-- result in failures that may not be significant.
110−serversUnequal :: (HasClient a, Testable (ShouldMatch (Client a)))
111− => Proxy a -> BaseUrl -> BaseUrl -> Int -> Expectation
112−serversUnequal api burl1 burl2 tries = do
113− mgr <- managerWithStoredReq
114− let args = stdArgs { chatty = False, maxSuccess = tries }
115− res <- quickCheckWithResult args $ serversEqualProperty api mgr burl1 burl2
116− case res of
117− Success _ _ _ -> prettyErr >>= expectationFailure
118− _ -> return ()
119−
120−serverDoesntSatisfy :: (HasClient a, Testable (ShouldSatisfy filt exp (Client a)))
121− => Proxy a -> BaseUrl -> Predicates filt -> Predicates exp
122− -> Int -> Expectation
123−serverDoesntSatisfy api burl filters expect tries = do
124− mgr <- managerWithStoredReq
125− let args = stdArgs { chatty = False, maxSuccess = tries }
126− res <- quickCheckWithResult args $ serverSatisfiesProperty api mgr burl filters expect
127− case res of
128− Success _ _ _ -> prettyErr >>= expectationFailure
129− _ -> return ()
130−
131−-- | Benchmarks a server with arbitrary requests using 'wrk'.
132−--
133−-- When using this, you should compile your program with '-threaded'.
134−-- Moreover, 'wrk' must be in the @$PATH@.
135−--
136−-- Note that this function is still very experimental, and it's behaviour will
137−-- likely change.
138−serverBenchmark ::
139− (HasClient a , Testable (ShouldSatisfy '[] '[Either ServantError Void] (Client a)))
140− => Proxy a -> BaseUrl -> BenchOptions -> IO ()
141−serverBenchmark api burl opts = replicateM_ (noOfTests opts) go
142− where
143− go = do
144− let alwaysTrue = addLeftPredicate (const True) emptyPredicates
145− serverSatisfies api burl emptyPredicates alwaysTrue 1
146− Just (r, _) <- readMVar currentReq
147− withSystemTempFile "wrkscript.lua" $ \f h -> do
148− let url = show $ getUri r
149− s = mkScript $ reqToWrk r
150− c = "wrk -c" ++ show (connections opts)
151− ++ " -d" ++ show (duration opts) ++ "s "
152− ++ " -t" ++ show (threads opts)
153− ++ " -s \"" ++ f ++ "\" "
154− ++ " --latency "
155− ++ url
156− hPutStrLn h s
157− hFlush h
158− callCommand c
159− -- While running wrk and the server on the same machine make the
160− -- results much less meaningful, this ameliorates the situation
161− -- somewhat.
162− performGC
163− threadDelay 1000
164
58
165−managerWithStoredReq :: IO Manager
166−managerWithStoredReq = newManager defaultManagerSettings { managerModifyRequest = go }
167− where go req = modifyMVar_ currentReq (addReq req) >> return req
168− addReq req _ = return $ Just (req, "")
src/Servant/QuickCheck/Internal/Testable.hsdeleted−111
@@ -1,111 +0,0 @@
1−-- | This module contains QuickCheck-related logic.
2−module Servant.QuickCheck.Internal.Testable where
3−
4−import Control.Concurrent.MVar (MVar, modifyMVar_, newMVar, readMVar)
5−import Control.Monad.Except (runExceptT, ExceptT)
6−import GHC.Generics (Generic)
7−import Network.HTTP.Client (Request, RequestBody (..),
8− requestBody, Manager)
9−import Servant.API ((:<|>)(..))
10−import Servant.Client (ServantError (..), BaseUrl)
11−import System.IO.Unsafe (unsafePerformIO)
12−import Test.QuickCheck (Arbitrary (..), discard)
13−import Test.QuickCheck.Property (Testable (..), forAllShrink,
14− ioProperty, (.&.))
15−
16−import Servant.QuickCheck.Internal.Predicates
17−
18−
19−type FinalClient a = Manager -> BaseUrl -> ExceptT ServantError IO a
20−
21−-- * ShouldMatch
22−
23−-- | Two corresponding client functions. Used for checking that APIs match.
24−data ShouldMatch a = ShouldMatch
25− { smClient :: a
26− , smManager :: Manager
27− , smBaseUrls :: (BaseUrl, BaseUrl)
28− } deriving (Functor, Generic)
29−
30−instance {-# OVERLAPPING #-} (Show a, Eq a)
31− => Testable (ShouldMatch (FinalClient a)) where
32− property sm = ioProperty $ do
33− let (burl1, burl2) = smBaseUrls sm
34− e1' <- runExceptT $ smClient sm (smManager sm) burl1
35− e2' <- runExceptT $ smClient sm (smManager sm) burl2
36− modifyMVar_ currentReq $ \x -> case x of
37− Nothing -> error "impossible"
38− Just (x', _) -> return $ Just (x', "LHS:\n" ++ show e1'
39− ++ "\nRHS:\n" ++ show e2')
40− case (e1', e2') of
41− (Right v1, Right v2) -> return $ v1 == v2
42− (Left (FailureResponse a1 b1 c1), Left (FailureResponse a2 b2 c2)) ->
43− return $ a1 == a2 && b1 == b2 && c1 == c2
44− (err1, err2) -> error $ "Exception response:"
45− ++ "\nLHS:\n" ++ show err1
46− ++ "\nRHS:\n" ++ show err2
47−
48−instance {-# OVERLAPPABLE #-} (Arbitrary a, Show a, Testable (ShouldMatch b))
49− => Testable (ShouldMatch (a -> b)) where
50− property sm = forAllShrink arbitrary shrink go
51− where go x = ($ x) <$> sm
52−
53−instance (Testable (ShouldMatch a), Testable (ShouldMatch b))
54− => Testable (ShouldMatch (a :<|> b)) where
55− property sm = property (fstAlt <$> sm) .&. property (sndAlt <$> sm)
56−
57−-- * ShouldSatisfy
58−
59−data ShouldSatisfy filter expect a = ShouldSatisfy
60− { ssVal :: a
61− , ssFilter :: Predicates filter
62− , ssExpect :: Predicates expect
63− , ssManager :: Manager
64− , ssBaseUrl :: BaseUrl
65− } deriving (Functor)
66−
67−instance {-# OVERLAPPING #-}
68− (Show a, Eq a, HasPredicate expect (Either ServantError a))
69− => Testable (ShouldSatisfy filter expect (FinalClient a)) where
70− property ss = ioProperty $ do
71− a' <- runExceptT $ ssVal ss (ssManager ss) (ssBaseUrl ss)
72− modifyMVar_ currentReq $ \x -> case x of
73− Nothing -> error "impossible"
74− Just (x', _) -> return $ Just (x', show a')
75− return $ getPredicate (ssExpect ss) a'
76−
77−instance {-# OVERLAPPABLE #-}
78− ( Arbitrary a, Show a, Testable (ShouldSatisfy filter expect b)
79− , HasPredicate filter a)
80− => Testable (ShouldSatisfy filter expect (a -> b)) where
81− property ss = forAllShrink arbitrary shrink go
82− where go x | getPredicate (ssFilter ss) x = ($ x) <$> ss
83− | otherwise = discard
84−
85−instance ( Testable (ShouldSatisfy filter expect a)
86− , Testable (ShouldSatisfy filter expect b))
87− => Testable (ShouldSatisfy filter expect (a :<|> b)) where
88− property ss = property (fstAlt <$> ss) .&. property (sndAlt <$> ss)
89−
90−-- * Utils
91−
92−fstAlt :: (a :<|> b) -> a
93−fstAlt (a :<|> _) = a
94−
95−sndAlt :: (a :<|> b) -> b
96−sndAlt (_ :<|> b) = b
97−
98−-- Used to store the current request and response so that in case of failure we
99−-- have the failing test in a user-friendly form.
100−currentReq :: MVar (Maybe (Request, String))
101−currentReq = unsafePerformIO $ newMVar Nothing
102−{-# NOINLINE currentReq #-}
103−
104−prettyErr :: IO String
105−prettyErr = do
106− Just (req, resp) <- readMVar currentReq
107− return $ show req ++ "Body:\n" ++ showReqBody (requestBody req)
108− ++ "\n\nResponse:\n" ++ resp
109− where
110− showReqBody (RequestBodyLBS x) = show x
111− showReqBody _ = error "expecting RequestBodyLBS"
test/Servant/QuickCheck/InternalSpec.hs+17−127
@@ -4,157 +4,47 @@ module Servant.QuickCheck.InternalSpec (spec) where
4
4
5import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
5import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
6import Control.Monad.IO.Class (liftIO)
6import Control.Monad.IO.Class (liftIO)
7+import Control.Monad (void)
8+import Network.HTTP.Client (newManager, defaultManagerSettings)
7import Data.Proxy
9import Data.Proxy
8import Servant
10import Servant
9import Test.Hspec
11import Test.Hspec
12+import Test.QuickCheck
13+import Test.QuickCheck.IO
14+import Test.QuickCheck.Monadic
10
15
11import Servant.QuickCheck.Internal
16import Servant.QuickCheck.Internal
12
17
13spec :: Spec
18spec :: Spec
14spec = do
19spec = do
15 serversEqualSpec
20 serversEqualSpec
16− serverSatisfiesSpec
17− serverBenchmarkSpec
18−
19
21
20serversEqualSpec :: Spec
22serversEqualSpec :: Spec
21serversEqualSpec = describe "serversEqual" $ do
23serversEqualSpec = describe "serversEqual" $ do
22
24
23− context "servers without function types" $ do
24−
25− it "considers equal servers equal" $ do
25+ it "considers equal servers equal" $ do
26− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
26+ mgr <- newManager defaultManagerSettings
27− serversEqual onlyReturnAPI burl burl noOfTestCases
28−
29− it "considers unequal servers unequal" $ do
30− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl1 ->
27+ withServantServer api server $ \burl1 ->
31− withServantServer onlyReturnAPI onlyReturnAPIServer' $ \burl2 ->
28+ withServantServer api server $ \burl2 -> do
32− serversUnequal onlyReturnAPI burl1 burl2 noOfTestCases
29+ return $ serversEqual api burl1 burl2 mgr
33−
34−
35− context "servers with function types" $ do
36−
37− it "considers equal servers equal" $ do
38− withServantServer functionAPI functionAPIServer $ \burl ->
39− serversEqual functionAPI burl burl noOfTestCases
40−
41− it "considers unequal servers unequal" $ do
42− withServantServer functionAPI functionAPIServer $ \burl1 ->
43− withServantServer functionAPI functionAPIServer' $ \burl2 ->
44− serversUnequal functionAPI burl1 burl2 noOfTestCases
45−
46−
47− context "stateful servers" $ do
48−
49− it "considers equal servers equal" $ do
50− withServantServer statefulAPI statefulAPIServer $ \burl1 ->
51− withServantServer statefulAPI statefulAPIServer $ \burl2 ->
52− serversEqual statefulAPI burl1 burl2 noOfTestCases
53−
54−
55−serverSatisfiesSpec :: Spec
56−serverSatisfiesSpec = describe "serverSatisfies" $ do
57−
58− it "passes true predicates" $ do
59− let e = addRightPredicate (== (5 :: Int)) emptyPredicates
60− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
61− serverSatisfies onlyReturnAPI burl emptyPredicates e noOfTestCases
62−
63− it "fails false predicates" $ do
64− let e = addRightPredicate (== (4 :: Int)) emptyPredicates
65− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
66− serverDoesntSatisfy onlyReturnAPI burl emptyPredicates e noOfTestCases
67−
68− it "allows filtering" $ do
69− let f = addPredicate (\(x :: String) -> length x > 2) emptyPredicates
70− e = addRightPredicate (\(x :: Int) -> x > 2) emptyPredicates
71− e' = addRightPredicate (\(x :: Int) -> x < 2) emptyPredicates
72− withServantServer functionAPI functionAPIServer $ \burl -> do
73− serverSatisfies functionAPI burl f e noOfTestCases
74− serverDoesntSatisfy functionAPI burl f e' noOfTestCases
75−
76− it "allows polymorphic predicates" $ do
77− let p1 x = length (show x) < 100000
78− p2 x = length (show x) < 1
79− e1 = addPolyPredicate (Proxy :: Proxy Show) p1 emptyPredicates
80− e2 = addPolyPredicate (Proxy :: Proxy Show) p2 emptyPredicates
81− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl -> do
82− serverSatisfies onlyReturnAPI burl emptyPredicates e1 noOfTestCases
83− serverDoesntSatisfy onlyReturnAPI burl emptyPredicates e2 noOfTestCases
84−
85
30
86− context "never500s" $ do
87
31
88− it "is true for servers that don't return 500s" $ do
89− withServantServer functionAPI functionAPIServer $ \burl ->
90− serverSatisfies functionAPI burl emptyPredicates never500s noOfTestCases
91
32
92− it "is false for servers that return 500s" $ do
93− withServantServer onlyReturnAPI onlyReturnAPIServer'' $ \burl ->
94− serverDoesntSatisfy onlyReturnAPI burl emptyPredicates never500s noOfTestCases
95−
96− context "onlyJsonObjects" $ do
97−
98− it "is false for servers that return top-level literals" $ do
99− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
100− serverDoesntSatisfy onlyReturnAPI burl emptyPredicates onlyJsonObjects noOfTestCases
101−
102−
103−serverBenchmarkSpec :: Spec
104−serverBenchmarkSpec = describe "serverBenchmark" $ do
105−
106− it "works" $ do
107− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
108− serverBenchmark onlyReturnAPI burl defaultBenchOptions
109
33
110------------------------------------------------------------------------------
34------------------------------------------------------------------------------
111-- APIs
35-- APIs
112------------------------------------------------------------------------------
36------------------------------------------------------------------------------
113
37
114−-- * OnlyReturn
115−
116−type OnlyReturnAPI = Get '[JSON] Int
117− :<|> Post '[JSON] String
118−
119−onlyReturnAPI :: Proxy OnlyReturnAPI
120−onlyReturnAPI = Proxy
121−
122−onlyReturnAPIServer :: IO (Server OnlyReturnAPI)
123−onlyReturnAPIServer = return $ return 5 :<|> return "hi"
124−
125−onlyReturnAPIServer' :: IO (Server OnlyReturnAPI)
126−onlyReturnAPIServer' = return $ return 5 :<|> return "hia"
127−
128−onlyReturnAPIServer'' :: IO (Server OnlyReturnAPI)
129−onlyReturnAPIServer'' = return $ error "err" :<|> return "hia"
130−
131−-- * Function
132−
133−type FunctionAPI = ReqBody '[JSON] String :> Post '[JSON] Int
134− :<|> Header "X-abool" Bool :> Get '[JSON] (Maybe Bool)
135−
136−functionAPI :: Proxy FunctionAPI
137−functionAPI = Proxy
138−
139−functionAPIServer :: IO (Server FunctionAPI)
140−functionAPIServer = return $ return . length :<|> return
141−
142−functionAPIServer' :: IO (Server FunctionAPI)
143−functionAPIServer'
144− = return $ (\x -> return $ length x - 1) :<|> \x -> return (not <$> x)
145−
146−-- * Stateful
147−
148−type StatefulAPI = ReqBody '[JSON] String :> Post '[JSON] String
38+type API = ReqBody '[JSON] String :> Post '[JSON] String
149− :<|> Get '[JSON] Int
39+ :<|> Get '[JSON] Int
150
40
151−statefulAPI :: Proxy StatefulAPI
41+api :: Proxy API
152−statefulAPI = Proxy
42+api = Proxy
153
43
154−statefulAPIServer :: IO (Server StatefulAPI)
44+server :: IO (Server API)
155−statefulAPIServer = do
45+server = do
156 mvar <- newMVar ""
46 mvar <- newMVar ""
157− return $ (\x -> liftIO $ swapMVar mvar x)
47+ return $ (\x -> liftIO $ print 'a' >> swapMVar mvar x)
158 :<|> (liftIO $ readMVar mvar >>= return . length)
48 :<|> (liftIO $ readMVar mvar >>= return . length)
159
49
160
50
test/Spec.hs+1−1
@@ -1 +1 @@
1−{-# OPTIONS_GHC -F -pgmF hspec-discover #-}
1+{-# OPTIONS_GHC -F -pgmF hspec-discover -optF --module-name=Spec #-}