on master help sign in
cleanup and example
commit4d0e2216aee53df522bb32d52394aad987aadaf3 authorMark Wotton <mwotton@gmail.com> dateThu 24 Dec 2020, 16:16 (6 years ago) parentb32a660
11 files changed+170−106
EXAMPLE.md+1 Example.lhs+112 README.md+5−30 package.yaml+10 roboservant.cabal+33−1 src/Roboservant/Types.hs−6 src/Roboservant/Types/Breakdown.hs+1−2 src/Roboservant/Types/BuildFrom.hs+3−16 src/Roboservant/Types/FlattenServer.hs−1 src/Roboservant/Types/ReifiedApi.hs−9 test/Spec.hs+5−41
EXAMPLE.mdnew file+1
@@ -0,0 +1 @@
1+Example.lhs
Example.lhsnew file+112
@@ -0,0 +1,112 @@
1+# Example
2+
3+Our api under test:
4+
5+```haskell
6+-- Obligatory fancy-types pragma tax
7+{-# LANGUAGE DataKinds #-}
8+{-# LANGUAGE TypeOperators #-}
9+{-# LANGUAGE DerivingStrategies #-}
10+{-# LANGUAGE StandaloneDeriving #-}
11+{-# LANGUAGE DerivingVia #-}
12+{-# LANGUAGE TypeApplications #-}
13+{-# LANGUAGE DeriveGeneric #-}
14+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
15+{-# LANGUAGE RecordWildCards #-}
16+
17+import Roboservant
18+import Test.Hspec
19+import Servant
20+import GHC.Generics
21+import Data.Typeable
22+import Data.Hashable
23+
24+newtype A = A Int
25+ deriving (Generic, Eq, Show, Typeable)
26+ deriving newtype (Hashable, FromHttpApiData, ToHttpApiData)
27+
28+newtype B = B Int
29+ deriving (Generic, Eq, Show, Typeable)
30+ deriving newtype (Hashable, FromHttpApiData, ToHttpApiData)
31+
32+type Api =
33+ "item" :> Get '[JSON] A
34+ :<|> "itemAdd" :> Capture "one" B :> Capture "two" B :> Get '[JSON] B
35+ :<|> "item" :> Capture "itemId" B :> Get '[JSON] ()
36+
37+goodServer = server (pure $ A 0)
38+badServer = server (pure $ A 1)
39+
40+server introduce = introduce :<|> combine :<|> eliminate
41+ where
42+ combine (B i) (B j) = pure $ B (i + j)
43+ eliminate (B i)
44+ | i > 10 = error "give up, eleven is way too big and probably not even real"
45+ | otherwise = pure ()
46+```
47+
48+Our tests would usually be in a separate file:
49+
50+```haskell
51+
52+
53+defaultConfig :: Config
54+defaultConfig = Config {
55+ -- you can pass extra values in using the seed argument. This can be useful
56+ -- for things that might not be produceable within the api, like auth tokens.
57+ seed = []
58+ , maxRuntime = 0.5
59+ -- if we get to 1000 interactions with the api, call it quits.
60+ , maxReps = 1000
61+ -- if you're using this inside quickcheck or hedgehog, you might want to set this
62+ -- from their seed to make sure it stays deterministic
63+ , rngSeed = 0
64+ -- 0 to 100: fail tests if we hit less than this percentage of endpoints.
65+ , coverageThreshold = 0
66+ }
67+
68+spec = describe "example" $ do
69+ it "good server should not fail" $ do
70+ fuzz @Api goodServer defaultConfig { coverageThreshold = 0.99 } (pure ())
71+ >>= (`shouldSatisfy` isNothing)
72+ it "bad server should fail" $ do
73+ fuzz @Api badServer defaultConfig { coverageThreshold = 0.99 } (pure ())
74+ >>= (`shouldSatisfy` serverFailure)
75+```
76+
77+And unless we want to ship roboservant and all its dependencies to production, we also need
78+some orphan instances: because As are the only value we can get without
79+an input, we need to be able to break them down.
80+
81+```haskell
82+deriving via (Compound A) instance Breakdown A
83+-- if we wanted to assemble As from parts as well, we'd derive using Compound
84+deriving via (Atom A) instance BuildFrom A
85+
86+```
87+
88+Similarly, to generate the first B from the Ints we got from inside the A, we need to be able to
89+build it up from components.
90+
91+```haskell
92+deriving via (Compound B) instance BuildFrom B
93+deriving via (Atom B) instance Breakdown B
94+
95+
96+main = hspec spec
97+```
98+
99+finally some uninteresting utilities
100+
101+```haskell
102+serverFailure :: Maybe Report -> Bool
103+serverFailure c = case c of
104+ Just Report{..} ->
105+ let RoboservantException{..} = rsException
106+ in failureReason /= NoPossibleMoves
107+ _ -> False
108+
109+isNothing x = case x of
110+ Nothing -> True
111+ _ -> False
112+```
README.md+5−30
@@ -4,43 +4,18 @@ Automatically fuzz your servant apis in a contextually-aware way.
44
55![CI](https://github.com/mwotton/roboservant/workflows/CI/badge.svg)
66
7+## example
78
8−# why?
9+see full example [here](EXAMPLE.md)
10+
11+## why?
912
1013Servant gives us a lot of information about what a server can do. We
1114use this information to generate arbitrarily long request/response
1215sessions and verify properties that should hold over them.
1316
14−# example
15−
16−Our api under test:
17−
18−```
19−newtype Foo = Foo Int
20− deriving (Generic, Eq, Show, Typeable)
21− deriving newtype (FromHttpApiData, ToHttpApiData)
22−
23−type FooApi =
24− "item" :> Get '[JSON] Foo
25− :<|> "itemAdd" :> Capture "one" Foo :> Capture "two" Foo :> Get '[JSON] Foo
26− :<|> "item" :> Capture "itemId" Foo :> Get '[JSON] ()
27−```
28−
29−From the tests:
30−
31−```
32− assert "should find an error in Foo" . not
33− =<< checkSequential (Group "Foo" [("Foo", RS.prop_sequential @Foo.FooApi Foo.fooServer)])
34−```
35−
36−We have a server that blows up if the value of the int in a `Foo` ever gets above 10. Note:
37−there is no generator for `Foo` types: larger `Foo`s can only be made only by combining existing
38−`Foo`s with `itemAdd`. This is an important distinction, because many APIs will return UUIDs or
39−similar as keys, which make it impossible to cover a useful section of the state space without
40−using the values returned by the API
41−
4217
43−# why not servant-quickcheck?
18+## why not servant-quickcheck?
4419
4520[servant-quickcheck](https://hackage.haskell.org/package/servant-quickcheck)
4621is a great package and I've learned a lot from it. Unfortunately, as mentioned previously,
package.yaml+10
@@ -54,3 +54,13 @@ tests:
5454 - hspec
5555 - hspec-core
5656 - http-api-data
57+
58+ example:
59+ main: Example.lhs
60+ source-dirs: .
61+
62+ dependencies:
63+ - roboservant
64+ - hspec
65+ build-depends: base, markdown-unlit
66+ ghc-options: -pgmL markdown-unlit
roboservant.cabal+33−1
@@ -4,7 +4,7 @@ cabal-version: 1.12
44--
55-- see: https://github.com/sol/hpack
66--
7−-- hash: 3b8eaf26233853cd94e20de7f84c25d6124cd919072681c3287a761526f82b9e
7+-- hash: 8de468df6e784f3723af9155bba9da9f43659b7d4e8e465fdac26eafcb0d4306
88
99name: roboservant
1010version: 0.1.0.2
@@ -62,6 +62,38 @@ library
6262 , vinyl
6363 default-language: Haskell2010
6464
65+test-suite example
66+ type: exitcode-stdio-1.0
67+ main-is: Example.lhs
68+ other-modules:
69+ Paths_roboservant
70+ hs-source-dirs:
71+ ./.
72+ ghc-options: -Wall -fwrite-ide-info -hiedir=.hie -pgmL markdown-unlit
73+ build-depends:
74+ base >=4.7 && <5
75+ , bytestring
76+ , containers
77+ , dependent-map
78+ , dependent-sum
79+ , hashable
80+ , hspec
81+ , lifted-base
82+ , monad-control
83+ , mtl
84+ , random
85+ , roboservant
86+ , servant
87+ , servant-client
88+ , servant-flatten
89+ , servant-server
90+ , string-conversions
91+ , text
92+ , time
93+ , unordered-containers
94+ , vinyl
95+ default-language: Haskell2010
96+
6597test-suite roboservant-test
6698 type: exitcode-stdio-1.0
6799 main-is: Spec.hs
src/Roboservant/Types.hs−6
@@ -2,15 +2,9 @@
22{-# LANGUAGE DataKinds #-}
33{-# LANGUAGE DerivingStrategies #-}
44{-# LANGUAGE FlexibleInstances #-}
5−{-# LANGUAGE GeneralizedNewtypeDeriving #-}
6−{-# LANGUAGE LambdaCase #-}
75{-# LANGUAGE PolyKinds #-}
86{-# LANGUAGE ScopedTypeVariables #-}
9−{-# LANGUAGE StandaloneDeriving #-}
10−{-# LANGUAGE TupleSections #-}
11−{-# LANGUAGE TypeApplications #-}
127{-# LANGUAGE TypeFamilies #-}
13−{-# LANGUAGE TypeOperators #-}
148{-# LANGUAGE UndecidableInstances #-}
159
1610module Roboservant.Types
src/Roboservant/Types/Breakdown.hs+1−2
@@ -31,7 +31,7 @@ class Breakdown x where
3131 breakdownExtras :: x -> [(Dynamic,Int)]
3232
3333instance (Hashable x, Typeable x) => Breakdown (Atom x) where
34− breakdownExtras (Atom x) = [(toDyn x, hash x)]
34+ breakdownExtras _ = []
3535
3636deriving via (Atom ()) instance Breakdown ()
3737deriving via (Atom Int) instance Breakdown Int
@@ -49,7 +49,6 @@ instance GBreakdown f => GBreakdown (M1 S c f ) where
4949instance GBreakdown b => GBreakdown (M1 D a b) where
5050 gBreakdownExtras (M1 f) = gBreakdownExtras f
5151
52−
5352instance GBreakdown b => GBreakdown (M1 C a b) where
5453 gBreakdownExtras (M1 f) = gBreakdownExtras f
5554
src/Roboservant/Types/BuildFrom.hs+3−16
@@ -9,13 +9,6 @@
99{-# LANGUAGE TypeOperators #-}
1010{-# LANGUAGE FlexibleInstances #-}
1111{-# LANGUAGE AllowAmbiguousTypes #-}
12−
13−
14−
15−
16−
17−
18−
1912module Roboservant.Types.BuildFrom where
2013
2114import Control.Monad(filterM)
@@ -29,8 +22,6 @@ import Roboservant.Types.Internal
2922import Data.Hashable
3023import qualified Data.IntSet as IntSet
3124import GHC.Generics
32−-- import Roboservant.Types.GBuildFrom(GBuildFrom(..))
33−
3425
3526buildFrom :: forall x . (Hashable x, BuildFrom x, Typeable x) => Stash -> Maybe (StashValue x)
3627buildFrom = buildStash . buildFrom'
@@ -73,9 +64,9 @@ deriving via (Compound (Maybe x)) instance (Typeable x, Hashable x, BuildFrom x)
7364-- this isn't wonderful, but we need a hand-rolled instance for recursive datatypes right now.
7465-- with an arbitrary-ish interface, we could use a size parameter, rng access etc.
7566instance (BuildFrom x) => BuildFrom [x] where
76− extras stash = map (\xs -> (concatMap fst xs,map snd xs)) $ powerset $ take 3 $ extras @x stash
67+ extras stash = map (\xs -> (concatMap fst xs,map snd xs)) $ powerset $ extras @x stash
7768 where
78− powerset xs = filterM (\_ -> [True, False]) xs
69+ powerset xs = filterM (const [True, False]) xs
7970
8071instance (Hashable x, Typeable x, Generic x, GBuildFrom (Rep x)) => BuildFrom (Compound (x::Type)) where
8172 extras stash = fmap (Compound . to) <$> gExtras stash
@@ -83,14 +74,13 @@ instance (Hashable x, Typeable x, Generic x, GBuildFrom (Rep x)) => BuildFrom (C
8374deriving via (Atom Int) instance BuildFrom Int
8475deriving via (Atom Char) instance BuildFrom Char
8576
86−
8777class GBuildFrom (f :: k -> *) where
8878 gExtras :: Stash -> [([Provenance], f a)]
8979
90−
9180instance GBuildFrom b => GBuildFrom (M1 D a b) where
9281 gExtras = fmap (fmap M1) . gExtras
9382
83+-- not recursion safe!
9484instance (GBuildFrom a, GBuildFrom b) => GBuildFrom (a :+: b) where
9585 gExtras stash = (fmap L1 <$> gExtras stash)
9686 <> (fmap R1 <$> gExtras stash)
@@ -98,9 +88,6 @@ instance (GBuildFrom a, GBuildFrom b) => GBuildFrom (a :+: b) where
9888instance (GBuildFrom a, GBuildFrom b) => GBuildFrom (a :*: b) where
9989 gExtras stash = [ (pa<>pb, a' :*: b') | (pa,a') <- gExtras stash , (pb,b') <- gExtras stash]
10090
101−
102−
103−
10491instance GBuildFrom b => GBuildFrom (M1 C a b) where
10592 gExtras =fmap (fmap M1) . gExtras
10693
src/Roboservant/Types/FlattenServer.hs−1
@@ -11,7 +11,6 @@ module Roboservant.Types.FlattenServer where
1111
1212import Servant
1313
14−
1514data Bundled endpoints where
1615 AnEndpoint :: Server endpoint -> Bundled endpoints -> Bundled (endpoint ': endpoints)
1716 NoEndpoints :: Bundled '[]
src/Roboservant/Types/ReifiedApi.hs−9
@@ -7,10 +7,7 @@
77{-# LANGUAGE GeneralizedNewtypeDeriving #-}
88{-# LANGUAGE LambdaCase #-}
99{-# LANGUAGE PolyKinds #-}
10−{-# LANGUAGE RecordWildCards #-}
1110{-# LANGUAGE ScopedTypeVariables #-}
12−{-# LANGUAGE StandaloneDeriving #-}
13−{-# LANGUAGE TupleSections #-}
1411{-# LANGUAGE TypeApplications #-}
1512{-# LANGUAGE TypeFamilies #-}
1613{-# LANGUAGE TypeOperators #-}
@@ -18,10 +15,8 @@
1815{-# LANGUAGE CPP #-}
1916module Roboservant.Types.ReifiedApi where
2017
21−
2218import Control.Monad.Except (runExceptT)
2319import Data.Bifunctor
24−-- import Data.Dependent.Sum
2520import Data.Dynamic (Dynamic)
2621import Data.Kind
2722import Data.List.NonEmpty (NonEmpty)
@@ -165,7 +160,6 @@ instance
165160 tagType (Argument (buildFrom @(IfRequiredLenient T.Text mods paramType)))
166161 V.:& reifiedEndpointArguments @endpoint
167162
168−
169163instance
170164 ( BuildFrom (IfRequiredLenient T.Text mods headerType)
171165 , ToReifiedEndpoint endpoint
@@ -178,8 +172,6 @@ instance
178172 tagType (Argument (buildFrom @(IfRequiredLenient T.Text mods headerType)))
179173 V.:& reifiedEndpointArguments @endpoint
180174
181−
182−-- this isn't happy in 0.16.2
183175#if MIN_VERSION_servant(0,17,0)
184176instance
185177 ( BuildFrom (IfLenient String mods captureType)
@@ -204,7 +196,6 @@ instance
204196 V.:& reifiedEndpointArguments @endpoint
205197
206198#endif
207−-- caching for merge
208199
209200instance
210201 ( BuildFrom (IfLenient String mods requestType)
test/Spec.hs+5−41
@@ -1,7 +1,6 @@
11{-# LANGUAGE OverloadedStrings #-}
22{-# LANGUAGE StandaloneDeriving #-}
33{-# LANGUAGE RecordWildCards #-}
4−{-# LANGUAGE DerivingVia #-}
54{-# LANGUAGE TypeApplications #-}
65{-# LANGUAGE LambdaCase #-}
76{-# LANGUAGE DerivingVia #-}
@@ -15,14 +14,13 @@ import qualified Post
1514import qualified Product
1615import qualified Breakdown
1716
18−import Test.Hspec.Core.Spec
19−
20−import Data.Dynamic(toDyn,Typeable,Dynamic)
17+import Test.Hspec.Core.Spec(ResultStatus(Failure,Success),resultStatus,itemExample,FailureReason(Reason),mapSpecItem_)
18+import Data.Dynamic(toDyn)
2119import qualified Roboservant as RS
2220import Test.Hspec
23−import Data.Void
24−import Data.Maybe
25−import Data.Hashable
21+import Data.Void ( Void )
22+import Data.Maybe ( isNothing )
23+import Data.Hashable ( Hashable(hash) )
2624
2725main :: IO ()
2826main = hspec spec
@@ -101,27 +99,6 @@ serverFailure = \case
10199 in failureReason /= RS.NoPossibleMoves
102100 _ -> False
103101
104− -- describe "can build from pieces" $ do
105− -- it "should find a failure that requires some assembly" $ do
106− -- RS.fuzz @RS.BuildFrom.Api RS.BuildFrom.server defaultConfig noCheck
107− -- >>= (`shouldSatisfy` isJust)
108−
109−
110−
111− -- -- -- The UnsafeIO checker does not actually really use the contextually aware stuff, though it
112− -- -- -- could: it's mostly here to show how to test for concurrency problems.
113− -- describe "concurrency bugs" $ do
114− -- before UnsafeIO.makeServer $ do
115− -- describe "sequential checking" $ do
116− -- it "safe use" $ \unsafeServer -> do
117− -- hedgehog $ RS.prop_sequential @UnsafeIO.UnsafeApi unsafeServer []
118−
119− -- modifyMaxSuccess (const 10000) $
120− -- shouldFail $
121− -- describe "concurrent" $ do
122− -- it "concurrent, dangerous use" $ \unsafeServer -> do
123− -- RS.prop_concurrent @UnsafeIO.UnsafeApi unsafeServer []
124−
125102
126103deriving via (RS.Atom Foo.Foo) instance RS.Breakdown Foo.Foo
127104deriving via (RS.Atom Foo.Foo) instance RS.BuildFrom Foo.Foo
@@ -144,19 +121,6 @@ deriving via (RS.Compound Product.Foo) instance RS.BuildFrom Product.Foo
144121deriving via (RS.Compound Breakdown.Foo) instance RS.Breakdown Breakdown.Foo
145122deriving via (RS.Compound Breakdown.SomeSum) instance RS.Breakdown Breakdown.SomeSum
146123
147−
148−
149−--deriving via (Compound RS.BuildFrom.Wrapped) instance RS.BuildFrom RS.BuildFrom.Wrapped
150−--deriving via (Compound RS.BuildFrom.Wrapped) instance RS.Breakdown RS.BuildFrom.Wrapped
151−
152−
153−
154−-- deriving via (RS.Atom Void) instance RS.BuildFrom Void
155−
156−-- instance RS.Breakdown Post.FooPost
157−-- instance RS.BuildFrom Post.FooPost
158−
159−
160124-- | `shouldFail` allows you to assert that a given `Spec` should contain at least one failing test.
161125-- this is often useful when testing tests.
162126shouldFail :: SpecWith a -> SpecWith a