git.iko.soy/servant-quickcheck
on master help sign in
Start implementing better error messages.
commit70fed09866b17a141aff154b5084f9dedb2be16b authorJulian K. Arni <jkarni@gmail.com> dateMon 1 Aug 2016, 11:58 (10 years ago) parent65a0809
4 files changed+99−15
servant-quickcheck.cabal+5 src/Servant/QuickCheck/Internal/ErrorTypes.hs+37 src/Servant/QuickCheck/Internal/QuickCheck.hs+11−10 test/Servant/QuickCheck/InternalSpec.hs+46−5
servant-quickcheck.cabal+5
@@ -45,6 +45,7 @@ library
4545 , case-insensitive == 1.2.*
4646 , hspec == 2.2.*
4747 , text == 1.*
48+ , pretty == 1.1.*
4849 if impl(ghc < 7.10)
4950 build-depends: bifunctors == 5.*
5051
@@ -77,6 +78,7 @@ test-suite spec
7778 , base-compat
7879 , servant-quickcheck
7980 , hspec
81+ , hspec-core
8082 , http-client
8183 , warp
8284 , servant-server
@@ -88,7 +90,10 @@ test-suite spec
8890 default-extensions: TypeOperators
8991 , FlexibleInstances
9092 , FlexibleContexts
93+ , GADTs
9194 , DataKinds
9295 , NoImplicitPrelude
96+ , OverloadedStrings
97+ , ScopedTypeVariables
9398 if flag(long-tests)
9499 cpp-options: -DLONG_TESTS
src/Servant/QuickCheck/Internal/ErrorTypes.hsnew file+37
@@ -0,0 +1,37 @@
1+module Servant.QuickCheck.Internal.ErrorTypes where
2+
3+import Text.PrettyPrint
4+
5+data Request = Request
6+ { requestBody :: String
7+ , requestHeaders :: [String]
8+ , requestUrl :: String
9+ } deriving (Eq, Show, Read, Generic)
10+
11+prettyReq :: Doc
12+prettyReq r =
13+ text "Request:" $ nest 5 $
14+ text "URL:" <+> text (nest 5 $ requestUrl r)
15+ $$ text "Headers:" <+>
16+ $$ text "Body:" <+> text (nest 5 $ requestBody r)
17+
18+instance IsString Request where
19+ fromString url = Request "" [] url
20+
21+data Response = Response
22+ { responseBody :: String
23+ , responseHeaders :: [String]
24+ } deriving (Eq, Show, Read, Generic)
25+
26+instance IsString Response where
27+ fromString body = Response body []
28+
29+-- The error that occurred.
30+data Failure
31+ = PredicateFailure String Request Response
32+ | ServerEqualityFailure Request Response Response
33+ deriving (Eq, Show, Read, Generic)
34+
35+instance Show Failure where
36+ show (PredicateFailure pred req resp)
37+ = "Predicate failed for " <> pred <> "
src/Servant/QuickCheck/Internal/QuickCheck.hs+11−10
@@ -2,10 +2,10 @@ module Servant.QuickCheck.Internal.QuickCheck where
22
33import qualified Data.ByteString.Lazy as LBS
44import Data.Proxy (Proxy)
5+import Data.String (IsString (..))
56import Data.Text (Text)
6−import Network.HTTP.Client (Manager, Request, checkStatus,
7− defaultManagerSettings, httpLbs,
8− newManager)
7+import GHC.Generics (Generic)
8+import qualified Network.HTTP.Client as C
99import Network.Wai.Handler.Warp (withApplication)
1010import Prelude.Compat
1111import Servant (Context (EmptyContext), HasServer,
@@ -17,9 +17,10 @@ import Test.QuickCheck (Args (..), Result (..),
1717 quickCheckWithResult)
1818import Test.QuickCheck.Monadic (assert, forAllM, monadicIO, run)
1919
20+import Servant.QuickCheck.Internal.Equality
2021import Servant.QuickCheck.Internal.HasGenRequest
2122import Servant.QuickCheck.Internal.Predicates
22−import Servant.QuickCheck.Internal.Equality
23+import Servant.QuickCheck.Internal.ErrorTypes
2324
2425
2526-- | Start a servant application on an open port, run the provided function,
@@ -58,8 +59,8 @@ serversEqual :: HasGenRequest a =>
5859serversEqual api burl1 burl2 args req = do
5960 let reqs = (\f -> (f burl1, f burl2)) <$> genRequest api
6061 r <- quickCheckWithResult args $ monadicIO $ forAllM reqs $ \(req1, req2) -> do
61− resp1 <- run $ httpLbs (noCheckStatus req1) defManager
62− resp2 <- run $ httpLbs (noCheckStatus req2) defManager
62+ resp1 <- run $ C.httpLbs (noCheckStatus req1) defManager
63+ resp2 <- run $ C.httpLbs (noCheckStatus req2) defManager
6364 assert $ getResponseEquality req resp1 resp2
6465 case r of
6566 Success {} -> return ()
@@ -116,9 +117,9 @@ serverDoesntSatisfy api burl args preds = do
116117 NoExpectedFailure {} -> expectationFailure $ "No expected failure"
117118 InsufficientCoverage {} -> expectationFailure $ "Insufficient coverage"
118119
119−noCheckStatus :: Request -> Request
120−noCheckStatus r = r { checkStatus = \_ _ _ -> Nothing}
120+noCheckStatus :: C.Request -> C.Request
121+noCheckStatus r = r { C.checkStatus = \_ _ _ -> Nothing}
121122
122−defManager :: Manager
123−defManager = unsafePerformIO $ newManager defaultManagerSettings
123+defManager :: C.Manager
124+defManager = unsafePerformIO $ C.newManager C.defaultManagerSettings
124125{-# NOINLINE defManager #-}
test/Servant/QuickCheck/InternalSpec.hs+46−5
@@ -1,5 +1,4 @@
1−{-# LANGUAGE CPP #-}
2−{-# LANGUAGE ScopedTypeVariables #-}
1+{-# LANGUAGE CPP #-}
32module Servant.QuickCheck.InternalSpec (spec) where
43
54import Control.Concurrent.MVar (newMVar, readMVar,
@@ -8,11 +7,16 @@ import Control.Monad.IO.Class (liftIO)
87import Prelude.Compat
98import Servant
109import Servant.API.Internal.Test.ComprehensiveAPI (comprehensiveAPI)
11−import Test.Hspec (Spec, describe, it,
12− shouldBe)
10+import Test.Hspec (Spec, context,
11+ describe, it,
12+ pending, shouldBe)
13+import Test.Hspec.Core.Spec (Arg, Example,
14+ Result (..),
15+ defaultParams,
16+ evaluateExample)
1317
1418import Servant.QuickCheck
15−import Servant.QuickCheck.Internal (genRequest, serverDoesntSatisfy)
19+import Servant.QuickCheck.Internal (genRequest, Failure(..), serverDoesntSatisfy)
1620
1721spec :: Spec
1822spec = do
@@ -28,6 +32,23 @@ serversEqualSpec = describe "serversEqual" $ do
2832 withServantServerAndContext api ctx server $ \burl2 -> do
2933 serversEqual api burl1 burl2 args bodyEquality
3034
35+ context "when servers are not equal" $ do
36+
37+ it "provides the failing requests in the error message" $ do
38+ Fail _ err <- withServantServer api2 server2 $ \burl1 ->
39+ withServantServer api2 server3 $ \burl2 -> do
40+ evalExample $ serversEqual api2 burl1 burl2 args bodyEquality
41+ let ServerEqualityFailure req _ _ = read err
42+ req `shouldBe` "failplz"
43+
44+ it "provides the failing responses in the error message" $ do
45+ Fail _ err <- withServantServer api2 server2 $ \burl1 ->
46+ withServantServer api2 server3 $ \burl2 -> do
47+ evalExample $ serversEqual api2 burl1 burl2 args bodyEquality
48+ let ServerEqualityFailure _ r1 r2 = read err
49+ r1 `shouldBe` "1"
50+ r2 `shouldBe` "2"
51+
3152
3253serverSatisfiesSpec :: Spec
3354serverSatisfiesSpec = describe "serverSatisfies" $ do
@@ -46,6 +67,9 @@ serverSatisfiesSpec = describe "serverSatisfies" $ do
4667 <%> notAllowedContainsAllowHeader
4768 <%> mempty)
4869
70+ context "when predicates are false" $
71+ it "fails with informative error messages" $ pending
72+
4973isComprehensiveSpec :: Spec
5074isComprehensiveSpec = describe "HasGenRequest" $ do
5175
@@ -72,12 +96,29 @@ server = do
7296 :<|> (liftIO $ readMVar mvar >>= return . length)
7397 :<|> (const $ return ())
7498
99+
100+type API2 = "failplz" :> Get '[JSON] Int
101+
102+api2 :: Proxy API2
103+api2 = Proxy
104+
105+server2 :: IO (Server API2)
106+server2 = return $ return 1
107+
108+server3 :: IO (Server API2)
109+server3 = return $ return 2
110+
75111ctx :: Context '[BasicAuthCheck ()]
76112ctx = BasicAuthCheck (const . return $ NoSuchUser) :. EmptyContext
77113------------------------------------------------------------------------------
78114-- Utils
79115------------------------------------------------------------------------------
80116
117+evalExample :: (Example e, Arg e ~ ()) => e -> IO Result
118+evalExample e = evaluateExample e defaultParams ($ ()) progCallback
119+ where
120+ progCallback _ = return ()
121+
81122args :: Args
82123args = defaultArgs { maxSuccess = noOfTestCases }
83124