git.iko.soy/servant-quickcheck
on master help sign in
34 files changed+2978−954
since splitsnapshots
unifiedsplit
.gitignore+5 .travis.yml+154−30 CHANGELOG.yaml+214 README.md+25 cabal.haskell-ci+7 cabal.project+4 doc/Main.hs+4 doc/ServersEqual.lhs+50−91 doc/doc.cabal+45−5 doc/posts/Announcement.anansi+330 doc/posts/LICENSE+30 doc/posts/Makefile+25 doc/posts/Setup.hs+2 doc/posts/announcement.md+318 doc/posts/posts.cabal+47 doc/posts/src/Main.hs+71 doc/posts/src/Spec.hs+40 doc/posts/src/schema.sql+9 doc/posts/stack.yaml+46 example/Main.hs+31 servant-quickcheck.cabal+126−76 src/Servant/QuickCheck.hs+63−69 src/Servant/QuickCheck/Internal.hs+5−4 src/Servant/QuickCheck/Internal/Benchmarking.hs−87 src/Servant/QuickCheck/Internal/Equality.hs+58 src/Servant/QuickCheck/Internal/ErrorTypes.hs+82 src/Servant/QuickCheck/Internal/HasGenRequest.hs+197 src/Servant/QuickCheck/Internal/Predicates.hs+453−124 src/Servant/QuickCheck/Internal/QuickCheck.hs+161−127 src/Servant/QuickCheck/Internal/Testable.hs−96 stack.yaml+5−31 test/Doctest.hs−44 test/Servant/CoMock/InternalSpec.hs−170 test/Servant/QuickCheck/InternalSpec.hs+371
.gitignore+5
@@ -1 +1,6 @@
11doc/_build/
2+scripts/
3+samples/
4+test-servers/
5+/doc/
6+.stack-work/
.travis.yml+154−30
@@ -1,34 +1,158 @@
1−sudo: false
2−
1+# This Travis job script has been generated by a script via
2+#
3+# haskell-ci '--config=cabal.haskell-ci' 'servant-quickcheck.cabal'
4+#
5+# To regenerate the script (for example after adjusting tested-with) run
6+#
7+# haskell-ci regenerate
8+#
9+# For more information, see https://github.com/haskell-CI/haskell-ci
10+#
11+# version: 0.10.1
12+#
13+version: ~> 1.0
314language: c
4−
5−env:
6− - GHCVER=7.8.4
7− - GHCVER=7.10.2
8−
9−addons:
10− apt:
11− sources:
12− - hvr-ghc
13− packages:
14− - ghc-7.8.4
15− - ghc-7.10.2
16− - cabal-install-1.22
17− - libgmp-dev
18− - wrk
19−
15+os: linux
16+dist: xenial
17+git:
18+ # whether to recursively clone submodules
19+ submodules: false
20+branches:
21+ only:
22+ - master
23+cache:
24+ directories:
25+ - $HOME/.cabal/packages
26+ - $HOME/.cabal/store
27+ - $HOME/.hlint
28+before_cache:
29+ - rm -fv $CABALHOME/packages/hackage.haskell.org/build-reports.log
30+ # remove files that are regenerated by 'cabal update'
31+ - rm -fv $CABALHOME/packages/hackage.haskell.org/00-index.*
32+ - rm -fv $CABALHOME/packages/hackage.haskell.org/*.json
33+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.cache
34+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar
35+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar.idx
36+ - rm -rfv $CABALHOME/packages/head.hackage
37+jobs:
38+ include:
39+ - compiler: ghc-8.8.3
40+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.8.3","cabal-install-3.2"]}}
41+ os: linux
42+ - compiler: ghc-8.6.5
43+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.6.5","cabal-install-3.2"]}}
44+ os: linux
45+ - compiler: ghc-8.4.4
46+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.4.4","cabal-install-3.2"]}}
47+ os: linux
48+ - compiler: ghc-8.2.2
49+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.2.2","cabal-install-3.2"]}}
50+ os: linux
51+ - compiler: ghc-8.0.2
52+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.0.2","cabal-install-3.2"]}}
53+ os: linux
54+before_install:
55+ - HC=$(echo "/opt/$CC/bin/ghc" | sed 's/-/\//')
56+ - WITHCOMPILER="-w $HC"
57+ - HADDOCK=$(echo "/opt/$CC/bin/haddock" | sed 's/-/\//')
58+ - HCPKG="$HC-pkg"
59+ - unset CC
60+ - CABAL=/opt/ghc/bin/cabal
61+ - CABALHOME=$HOME/.cabal
62+ - export PATH="$CABALHOME/bin:$PATH"
63+ - TOP=$(pwd)
64+ - "HCNUMVER=$(${HC} --numeric-version|perl -ne '/^(\\d+)\\.(\\d+)\\.(\\d+)(\\.(\\d+))?$/; print(10000 * $1 + 100 * $2 + ($3 == 0 ? $5 != 1 : $3))')"
65+ - echo $HCNUMVER
66+ - CABAL="$CABAL -vnormal+nowrap"
67+ - set -o pipefail
68+ - TEST=--enable-tests
69+ - BENCH=--enable-benchmarks
70+ - HEADHACKAGE=false
71+ - rm -f $CABALHOME/config
72+ - |
73+ echo "verbose: normal +nowrap +markoutput" >> $CABALHOME/config
74+ echo "remote-build-reporting: anonymous" >> $CABALHOME/config
75+ echo "write-ghc-environment-files: always" >> $CABALHOME/config
76+ echo "remote-repo-cache: $CABALHOME/packages" >> $CABALHOME/config
77+ echo "logs-dir: $CABALHOME/logs" >> $CABALHOME/config
78+ echo "world-file: $CABALHOME/world" >> $CABALHOME/config
79+ echo "extra-prog-path: $CABALHOME/bin" >> $CABALHOME/config
80+ echo "symlink-bindir: $CABALHOME/bin" >> $CABALHOME/config
81+ echo "installdir: $CABALHOME/bin" >> $CABALHOME/config
82+ echo "build-summary: $CABALHOME/logs/build.log" >> $CABALHOME/config
83+ echo "store-dir: $CABALHOME/store" >> $CABALHOME/config
84+ echo "install-dirs user" >> $CABALHOME/config
85+ echo " prefix: $CABALHOME" >> $CABALHOME/config
86+ echo "repository hackage.haskell.org" >> $CABALHOME/config
87+ echo " url: http://hackage.haskell.org/" >> $CABALHOME/config
2088install:
21− - (mkdir -p $HOME/.local/bin && cd $HOME/.local/bin && wget https://zalora-public.s3.amazonaws.com/tinc && chmod +x tinc)
22− - export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/1.22/bin:$PATH
23− - ghc --version
24− - cabal --version
25− - travis_retry cabal update
26− - sed -i 's/^jobs:/-- jobs:/' ${HOME}/.cabal/config
27−
89+ - ${CABAL} --version
90+ - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]"
91+ - |
92+ echo "program-default-options" >> $CABALHOME/config
93+ echo " ghc-options: $GHCJOBS +RTS -M6G -RTS" >> $CABALHOME/config
94+ - cat $CABALHOME/config
95+ - rm -fv cabal.project cabal.project.local cabal.project.freeze
96+ - travis_retry ${CABAL} v2-update -v
97+ # Generate cabal.project
98+ - rm -rf cabal.project cabal.project.local cabal.project.freeze
99+ - touch cabal.project
100+ - |
101+ echo "packages: ." >> cabal.project
102+ - if [ $HCNUMVER -ge 80200 ] ; then echo 'package servant-quickcheck' >> cabal.project ; fi
103+ - "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
104+ - |
105+ - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(servant-quickcheck)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
106+ - cat cabal.project || true
107+ - cat cabal.project.local || true
108+ - if [ -f "./configure.ac" ]; then (cd "." && autoreconf -i); fi
109+ - ${CABAL} v2-freeze $WITHCOMPILER ${TEST} ${BENCH}
110+ - "cat cabal.project.freeze | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'"
111+ - rm cabal.project.freeze
112+ - travis_wait 40 ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} --dep -j2 all
113+ - travis_wait 40 ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --dep -j2 all
28114script:
29− - tinc && cabal configure --enable-tests && cabal build && cabal test
30− - (cd doc && tinc cabal configure --enable-tests && cabal build && cabal test)
115+ - DISTDIR=$(mktemp -d /tmp/dist-test.XXXX)
116+ # Packaging...
117+ - ${CABAL} v2-sdist all
118+ # Unpacking...
119+ - mv dist-newstyle/sdist/*.tar.gz ${DISTDIR}/
120+ - cd ${DISTDIR} || false
121+ - find . -maxdepth 1 -type f -name '*.tar.gz' -exec tar -xvf '{}' \;
122+ - find . -maxdepth 1 -type f -name '*.tar.gz' -exec rm '{}' \;
123+ - PKGDIR_servant_quickcheck="$(find . -maxdepth 1 -type d -regex '.*/servant-quickcheck-[0-9.]*')"
124+ # Generate cabal.project
125+ - rm -rf cabal.project cabal.project.local cabal.project.freeze
126+ - touch cabal.project
127+ - |
128+ echo "packages: ${PKGDIR_servant_quickcheck}" >> cabal.project
129+ - if [ $HCNUMVER -ge 80200 ] ; then echo 'package servant-quickcheck' >> cabal.project ; fi
130+ - "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
131+ - |
132+ - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(servant-quickcheck)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
133+ - cat cabal.project || true
134+ - cat cabal.project.local || true
135+ # Building...
136+ # this builds all libraries and executables (without tests/benchmarks)
137+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all
138+ # Building with tests and benchmarks...
139+ # build & run tests, build benchmarks
140+ - ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} all
141+ # Testing...
142+ - ${CABAL} v2-test $WITHCOMPILER ${TEST} ${BENCH} all
143+ # cabal check...
144+ - (cd ${PKGDIR_servant_quickcheck} && ${CABAL} -vnormal check)
145+ # haddock...
146+ - ${CABAL} v2-haddock $WITHCOMPILER --with-haddock $HADDOCK ${TEST} ${BENCH} all
147+ # Building without installed constraints for packages in global-db...
148+ - rm -f cabal.project.local
149+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all
150+ # Constraint sets
151+ - rm -rf cabal.project.local
152+ # Constraint set base-compat-0.10
153+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --constraint='base-compat ==0.10.*' all
154+ # Constraint set base-compat-0.11
155+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --constraint='base-compat ==0.11.*' all
31156
32−cache:
33− directories:
34− - $HOME/.tinc/cache
157+# REGENDATA ("0.10.1",["--config=cabal.haskell-ci","servant-quickcheck.cabal"])
158+# EOF
CHANGELOG.yamlnew file+214
@@ -0,0 +1,214 @@
1+releases:
2+ - version: "0.0.9.0"
3+ changes:
4+ - description: "Support servant-0.15 (#65)"
5+ authors: fizruk
6+ date: 2020-06-25
7+
8+ - description: "Relax constraints for GHC 8.10 (#70)"
9+ authors: felixmulder
10+ date: 2020-06-20
11+
12+ - version: "0.0.8.0"
13+ changes:
14+ - description: Support for servant-0.17
15+ authors: phadej
16+ date: 2019-01-23
17+
18+ - version: "0.0.7.3"
19+ changes:
20+ - description: Support for servant-0.14
21+ issue: 53
22+ authors: phadej
23+ date: 2018-06-12
24+
25+ - description: Fix a failure from OnlyJsonObjects when there is no content-type.
26+ issue: 55
27+ authors: Phenitei
28+ date: 2018-08-27
29+
30+ - description: A bug fix where notAllowedContainsAllowHeader would print the initial request alongside the failure instead of the request causing the failure when it failed.
31+ issue: 57
32+ authors: Phenitei
33+ date: 2018-08-29
34+
35+ - description: QuickCheck 2.12 compatibility
36+ issue: 58
37+ authors: parsonsmatt
38+ date: 2018-10-12
39+
40+ - description: GHC 8.6 compatibility
41+ issue: 59
42+ authors: phadej
43+ date: 2018-10-15
44+
45+ - version: "0.0.7.2"
46+ changes:
47+
48+ - description: Allow client to pass an HTTP Manager in to functions
49+ issue: 47
50+ authors: parsonsmatt
51+ date: 2018-05-10
52+
53+ - description: Fix "should not happen" error when exceptions are thrown
54+ issue: 48
55+ authors: parsonsmatt
56+ date: 2018-05-10
57+
58+ - version: "0.0.7.0"
59+ changes:
60+
61+ - description: Support for base-compat-0.10
62+ issue: none
63+ authors: phadej
64+ date: 2018-04-12
65+
66+ - version: "0.0.7.0"
67+ changes:
68+
69+ - description: Support for GHC-8.4.1
70+ issue: none
71+ authors: phadej
72+ date: 2018-03-23
73+
74+ - description: Requires hspec-2.5
75+ issue: none
76+ authors: phadej
77+ date: 2018-03-23
78+
79+ - version: "0.0.6.0"
80+ changes:
81+
82+ - description: Support for servant-0.12
83+ issue: none
84+ authors: phadej
85+ date: 2018-02-09
86+
87+ - version: "0.0.5.0"
88+ changes:
89+
90+ - description: Export forgotten predicates
91+ issue: none
92+ pr: 40
93+ authors: Phenitei
94+ date: 2017-12-14
95+
96+ - version: "0.0.4"
97+ changes:
98+
99+ - description: Support for Servant 0.12
100+ issue: none
101+ authors: phadej
102+ date: 2017-11-07
103+
104+ - description: Support for Servant 0.11
105+ issue: none
106+ pr: 32
107+ authors: adinapoli-iohk
108+ date: 2017-10-18
109+ notes: Includes 0-weighted instance for EmptyAPI
110+
111+ - version: "0.0.3.0"
112+ changes:
113+
114+ - description: Add jsonEquality
115+ issue: 2
116+ pr: 30
117+ authors: erewok
118+ date: 2017-10-15
119+
120+ - description: Support hspec >= 2.4
121+ issue: 27
122+ pr: 29
123+ authors: erewok, jkarni
124+ date: 2017-10-15
125+
126+ - version: "0.0.2.4"
127+ changes:
128+
129+ - description: Don't append slashes to paths
130+ issue: 22
131+ authors: declension
132+ date: 2017-03-11
133+
134+ - version: "0.0.2.3"
135+ changes:
136+
137+ - description: Fix QueryParam and QueryFlag requests
138+ issue: 23
139+ authors: declension
140+ date: 2017-03-10
141+
142+ - version: "0.0.2.2"
143+ changes:
144+
145+ - description: Make onlyJsonObjects succeed in non-JSON endpoints
146+ issue: 20
147+ authors: jkarni
148+ date: 2016-10-18
149+
150+
151+ - version: "0.0.2.1"
152+ changes:
153+
154+ - description: Add notLongerThan predicate
155+ pr: 17
156+ authors: jkarni
157+ date: 2016-10-05
158+
159+ - description: Add getsHaveLastModifiedHeader predicate
160+ pr: none
161+ authors: jkarni
162+ date: 2016-10-03
163+
164+ - description: Raise upper bounds
165+ notes: >
166+ For Quickcheck, aeson, http-client, servant, servant-client and
167+ servant-server.
168+ pr: none
169+ authors: jkarni
170+ date: 2016-10-03
171+
172+
173+ - version: "0.0.2.0"
174+ changes:
175+
176+ - description: Update CHANGELOG to YAML syntax.
177+ pr: 16
178+ authors: jkarni
179+ date: 2016-09-14
180+
181+ - description: Support new CaptureAll combinator
182+ pr: 16
183+ authors: jkarni
184+ date: 2016-09-14
185+
186+ - description: Support GHC 8
187+ pr: 16
188+ authors: jkarni
189+ date: 2016-09-14
190+
191+ - version: "0.0.1.1"
192+ changes:
193+
194+ - description: Exclude GHC 7.8 (by bumping base lower bound to 4.8)
195+ pr: none
196+ authors: jkarni
197+
198+ - description: More generous bounds for other packages.
199+ pr: none
200+ authors: jkarni
201+
202+ - version: "0.0.1.0"
203+ changes:
204+
205+ - description: Better error messages.
206+ notes: >
207+ Error messages now contain failing predicate, failing response and
208+ (except for response predicates), failing requests.
209+ pr: none
210+ authors: jkarni
211+
212+ - description: Signicant changes to RequestPredicate and ResponsePredicate types.
213+ pr: none
214+ authors: jkarni
README.mdnew file+25
@@ -0,0 +1,25 @@
1+# servant-quickcheck
2+
3+`servant-quickcheck` provides tools to test properties across entire APIs.
4+Rather than writing the same tests for each endpoint, with `servant-quickcheck`
5+you can simply specify properties every endpoint must meet once and for all.
6+For example:
7+
8+``` haskell
9+followsBestPractices :: Spec
10+followsBestPractices = describe "my API" $ do
11+
12+ it "follows best practices" $ do
13+ withServantServer myAPI myServer $ \burl ->
14+ serverSatisfies api burl stdArgs
15+ ( not500
16+ <%> onlyJsonObjects
17+ <%> getsHaveCacheControlHeader
18+ <%> headsHaveCacheControlHeader
19+ <%> mempty)
20+```
21+
22+Additionally, `servant-quickcheck` provides a `serversEqual` function that
23+generates arbitrary requests (that conform to the description of an API) and
24+tests that two servers respond identically to them. This can be useful when
25+refactoring or rewriting an API that should not change.
cabal.haskell-cinew file+7
@@ -0,0 +1,7 @@
1+branches: master
2+
3+constraint-set base-compat-0.10
4+ constraints: base-compat ==0.10.*
5+
6+constraint-set base-compat-0.11
7+ constraints: base-compat ==0.11.*
cabal.projectnew file+4
@@ -0,0 +1,4 @@
1+packages: .
2+tests: true
3+
4+allow-newer: servant-blaze:servant
doc/Main.hsnew file+4
@@ -0,0 +1,4 @@
1+import ServersEqual
2+
3+main :: IO ()
4+main = main1
doc/ServersEqual.lhs+50−91
@@ -20,98 +20,34 @@ don't need to know anything about Django or Python to follow along; indeed,
2020part of the fun of it is using `serversEqual` to guide you through
2121re-implementing code you may not entirely understand.
2222
23−Looking at the code, we can see the routes in `urls.py`:
24−
25−``` python
26−urlpatterns = patterns('',
27− url(r'^$', RedirectView.as_view(url='/todos')),
28− url(r'^todos$', views.TodoList.as_view()),
29− url(r'^todo/(?P<pk>[0-9]+)$', views.Todo.as_view()),
30−)
31−```
32−
33−And the handlers in `views.py`:
34−
35−``` python
36−class TodoList(APIView):
37− def get(self, request, format=None):
38− todo_items = TodoItem.objects.all()
39− serializer = TodoItemSerializer(todo_items, many=True)
40− return JSONResponse(serializer.data, status=status.HTTP_200_OK)
41−
42− def post(self, request, format=None):
43− serializer = TodoItemSerializer(data=request.DATA)
44− if serializer.is_valid():
45− saved_item = serializer.save()
46− saved_item.url = request.build_absolute_uri('/todo/' + str(saved_item.id))
47− saved_item.save()
48− serializer = TodoItemSerializer(instance=saved_item)
49− return JSONResponse(serializer.data, status=status.HTTP_201_CREATED)
50− return JSONResponse(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
51−
52− def delete(self, request, format=None):
53− TodoItem.objects.all().delete()
54− return JSONResponse(None, status=status.HTTP_204_NO_CONTENT)
55−
56−class Todo(APIView):
57− def get(self, request, pk, format=None):
58− try:
59− todoItem = TodoItem.objects.get(pk=pk)
60− serializer = TodoItemSerializer(todoItem)
61− except TodoItem.DoesNotExist:
62− return JSONResponse(None, status=status.HTTP_400_BAD_REQUEST)
63− return JSONResponse(serializer.data, status=status.HTTP_200_OK)
64−
65− def delete(self, request, pk, format=None):
66− try:
67− todoItem = TodoItem.objects.get(pk=pk)
68− todoItem.delete()
69− except TodoItem.DoesNotExist:
70− return JSONResponse(None, status=status.HTTP_400_BAD_REQUEST)
71− return JSONResponse(None, status=status.HTTP_204_NO_CONTENT)
72−
73− def patch(self, request, pk, format=None):
74− try:
75− todoItem = TodoItem.objects.get(pk=pk)
76− except TodoItem.DoesNotExist:
77− return JSONResponse(None, status=status.HTTP_400_BAD_REQUEST)
78− serializer = TodoItemSerializer(data=request.DATA, instance=todoItem, partial=True)
79− if serializer.is_valid():
80− serializer.save()
81− return JSONResponse(serializer.data, status=status.HTTP_200_OK)
82− return JSONResponse(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
83−```
84−
85−And from `models.py`:
86−
87−``` python
88−
89−class TodoItem(models.Model):
90− title = models.CharField(max_length=256, null=True, blank=True)
91− completed = models.NullBooleanField(null=True, blank=True, default=False)
92− url = models.CharField(max_length=256, null=True, blank=True)
93− order = models.IntegerField(null=True, blank=True)
94−
95−```
96−
97−So as a first pass, let's try:
23+Looking around the codebase (`urls.py`, `models.py` and `views.py` in
24+particular) may lead us to a first attempt at a rewrite:
9825
9926``` haskell
10027{-# LANGUAGE DataKinds #-}
28+{-# LANGUAGE DeriveAnyClass #-}
29+{-# LANGUAGE DeriveGeneric #-}
10130{-# LANGUAGE TypeOperators #-}
10231
32+module ServersEqual where
33+
34+import Control.Concurrent (forkIO, killThread)
35+import Control.Concurrent.STM as STM
36+import Control.Monad.IO.Class
37+import Data.Aeson (ToJSON, FromJSON)
38+import Data.IntMap as M
39+import GHC.Generics (Generic)
40+import Network.Wai.Handler.Warp (run)
10341import Servant
104−import Servant.QuickCheck
105−import STMContainers.Map as M
106−import GHC.Conc (atomically)
107−import Test.QuickCheck
42+import Servant.QuickCheck (serversEqual, BaseUrl(..), Scheme(..))
43+import Test.QuickCheck (Arbitrary(..))
10844
10945data Todo = Todo
11046 { title :: String
11147 , completed :: Bool
11248 , url :: String
11349 , order :: Int
114− } deriving (Eq, Show, Read)
50+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
11551
11652type API = TodosAPI :<|> TodoAPI
11753
@@ -125,30 +61,44 @@ type TodoAPI
12561 = "todo" :> Capture "id " Int :>
12662 ( Get '[JSON] Todo
12763 :<|> ReqBody '[JSON] Todo :> Patch '[JSON] ()
128− :<|> Delete '[JSON} ())
64+ :<|> Delete '[JSON] ())
12965
130−serverTodos :: Server TodosAPI
66+serverTodos :: DB -> Server TodosAPI
13167serverTodos tvar = getTodos tvar
13268 :<|> postTodos tvar
13369 :<|> deleteAllTodos tvar
13470
71+{-
13572serverTodo :: Server TodoAPI
13673serverTodo id' = getTodo tvar id'
13774 :<|> patchTodo tvar id'
13875 :<|> deleteTodo tvar id'
76+-}
77+
78+type DB = TVar (M.IntMap Todo)
13979
140−getTodos :: Map Int Todo -> Handler [Todo]
141−getTodos m = liftIO . atomically . toList $ S.stream m
80+getTodos :: DB -> Handler [Todo]
81+getTodos m = liftIO $ M.elems <$> STM.readTVarIO m
14282
143−postTodos :: Map Int Todo -> Todo -> Handler ()
144−postTodos m t = liftIO . atomically $ S.insert m t
83+postTodos :: DB -> Todo -> Handler ()
84+postTodos m t = liftIO . STM.atomically $ STM.modifyTVar' m (M.insert (order t) t)
14585
146−deleteTodos :: Map Int Todo -> Todo -> Handler ()
147−deleteTodos m t = liftIO . atomically $ S.insert m t
86+deleteAllTodos :: DB -> Handler ()
87+deleteAllTodos m = liftIO . STM.atomically $ STM.writeTVar m M.empty
88+
89+main1 :: IO ()
90+main1 = do
91+ db <- STM.newTVarIO M.empty
92+ tId <- forkIO $ run 8000 $ serve (Proxy :: Proxy TodosAPI) $ serverTodos db
93+ _ <- serversEqual (Proxy :: Proxy TodosAPI)
94+ (BaseUrl Http "localhost" 8000 "")
95+ (BaseUrl Http "localhost" 8001 "")
96+ 1000
97+ killThread tId
14898```
14999
150−(We're keeping the `Todo`s in memory for simplicity - if this were a production
151− application, we'd likely want to use a database.)
100+(We're keeping the `Todo`s in an `MVar` for simplicity. If this were a
101+ production application, we'd likely want to use a database.)
152102
153103Notice that we split up the API into two sub-APIs. Partly this makes things
154104cleaner and more readable, but there's also a more concrete benefit: we can
@@ -160,7 +110,16 @@ In order to check how we're doing, we need to add an `Arbitrary` instance for
160110
161111``` haskell
162112instance Arbitrary Todo where
163− arbitrary = Todo <$> arbitrary <$> arbitrary <$> arbitrary <$> arbitrary
113+ arbitrary = Todo <$> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary
164114```
165115
116+Let's try it out. First we need to get the old application running:
117+
118+``` bash
119+git clone https://github.com/mihirk/todo-backend-django.git
120+cd todo-backend-django
121+virtualenv venv
122+. ./venv/bin/activate
123+python manage.py runserver localhost:8001
124+```
166125
doc/doc.cabal+45−5
@@ -7,11 +7,51 @@ maintainer: jkarni@gmail.com
77build-type: Simple
88cabal-version: >=1.10
99
10−library
11− exposed-modules: ServersEqual
12− other-extensions: DataKinds, TypeOperators
10+-- library
11+-- exposed-modules: ServersEqual
12+-- other-extensions: DataKinds, TypeOperators
13+-- build-depends: base >=4.8 && <4.9
14+-- , servant-server == 0.7.*
15+-- , servant-quickcheck
16+-- , servant-client
17+-- , QuickCheck
18+-- , stm
19+-- , containers
20+-- , transformers
21+-- , warp
22+-- , aeson
23+-- ghc-options: -Wall -Werror -pgmL markdown-unlit
24+-- default-language: Haskell2010
25+--
26+-- executable doc
27+-- main-is: Main.hs
28+-- build-depends: base >=4.8 && <4.9
29+-- , servant-server == 0.7.*
30+-- , servant-quickcheck
31+-- , servant-client
32+-- , QuickCheck
33+-- , stm
34+-- , containers
35+-- , transformers
36+-- , warp
37+-- , aeson
38+-- default-language: Haskell2010
39+-- ghc-options: -Wall -Werror -pgmL markdown-unlit
40+
41+executable announcement
42+ main-is: Announcement.lhs
1343 build-depends: base >=4.8 && <4.9
1444 , servant-server == 0.7.*
15−
16− ghc-options: -Wall -Werror -pgmL markdown-unlit
45+ , servant-quickcheck
46+ , servant-client
47+ , QuickCheck
48+ , stm
49+ , containers
50+ , transformers
51+ , warp
52+ , aeson
53+ , hspec
54+ , postgresql-simple
55+ , text
1756 default-language: Haskell2010
57+ ghc-options: -Wall
doc/posts/Announcement.anansinew file+330
@@ -0,0 +1,330 @@
1+:loom anansi.markdown
2+
3+# Announcing servant-quickcheck
4+
5+Some time ago, we released `servant-mock`. The idea behind it is to use
6+`QuickCheck` to create a mock server that accords with a servant API. Not long
7+after, we started thinking about an analog that would, instead of mocking a
8+server, mock a client instead - i.e., generate random requests that conform to
9+an API description.
10+
11+This is much closer to the traditional use of `QuickCheck`. The most obvious
12+use-case is checking that properties hold of an *entire* server rather than of
13+individual endpoints. (But there are other uses that you can skip to if they
14+sound more interesting.)
15+
16+## `serverSatisfies`
17+
18+A useful guideline when writing and maintaing software is that, if there isn't
19+a test for a behaviour or property, sooner or later that property will be broken.
20+Another important perspective is that tests are a form of documentation - the
21+present developer telling future ones "this matters, and should be this way".
22+
23+The advantage of using tests for this form of documentation is that there's
24+simply too much information to convey, some of it only relevant to very specific
25+use cases, and rather than overload developers with an inexhaustible quantity of
26+details that would be hard to keep track of or remember, tests are a mechanism
27+of reminding developers of *only the relevant information, at the right time*.
28+<<EXAMPLE>>.
29+
30+We might hope that we could use tests to communicate the wide array of best
31+practices that have developed around APIs. About to return a top-level integer
32+in JSON? A test should say that's bad practice. About to not catch exceptions
33+and give a more meaningful HTTP status code? Another test there to stop you.
34+
35+Traditionally, in web services these things get done at the level of *individual*
36+endpoints. But this means that if a developer who hasn't had extensive experience with web
37+programming best practices writes a *new* endpoint which *does* return a top-level
38+integer literal, there's no test there to stop her. Code review might help, but
39+code review is much more error prone than tests, and really only meant for those
40+things that are too subtle to automate. (Indeed, if code review were such a reliable
41+defense mechanism against bugs and bad code, why have tests and linters at all?)
42+
43+The problem, then, with thinking about tests as only existing at the level of individual
44+endpoints is that there are no tests *for* tests - tests that check that new
45+behaviour and tests conforms to higher-level, more general best practices.
46+
47+`servant-quickcheck` aims to solve that. It allows describing properties that
48+*all* endpoints must satisfy. If a new endpoint comes along, it too will be
49+tested for that property, without any further work.
50+
51+Why isn't this idea already popular? Well, most web frameworks don't have a
52+reified description of APIs (beyond perhaps the routes). When you don't know
53+what the endpoints of an application are, and what request body they expect,
54+trying to generate arbitrary requests is almost entirely going to result in
55+404s (not found) and 400s (bad request). Maybe one in a thousand requests will
56+actually test a handler. Not very useful.
57+
58+`servant` applications, on the other hand, have a machine-readable API description
59+already available. And they already associate "correct" requests with particular
60+types. It's a small step, therefore, to generate 'arbitrary' values for these
61+requests, and all of them will go through to your handlers. (Note: all of the
62+uses of `servant-quickcheck` work with applications *not* written with servant-server -
63+and indeed not *in Haskell - but the API must be described with the servant
64+DSL.)
65+
66+Let's see how this works in practice. As a running example, let's use a simple
67+service that allows adding, removing, and querying biological species. Our SQL
68+schema is:
69+
70+:d schema.sql
71+
72+CREATE TABLE genus (
73+ genus_name text PRIMARY KEY,
74+ genus_family text NOT NULL
75+);
76+
77+CREATE TABLE species (
78+ species_name text PRIMARY KEY,
79+ species_genus text NOT NULL REFERENCES genus (genus_name)
80+)
81+:
82+
83+And our actual application:
84+
85+:d Main.hs
86+{-# LANGUAGE DataKinds #-}
87+{-# LANGUAGE DeriveAnyClass #-}
88+{-# LANGUAGE DeriveGeneric #-}
89+{-# LANGUAGE TypeOperators #-}
90+{-# LANGUAGE OverloadedStrings #-}
91+{-# LANGUAGE RecordWildCards #-}
92+module Main where
93+
94+import Servant
95+import Data.Aeson
96+import Database.PostgreSQL.Simple
97+import GHC.Generics (Generic)
98+import Data.Text (Text)
99+import Network.Wai.Handler.Warp
100+import Control.Monad.IO.Class (liftIO)
101+
102+type API
103+ = "species" :> (Capture "species-name" Text :> ( Get '[JSON] Species
104+ :<|> Delete '[JSON] ())
105+ :<|> ReqBody '[JSON] Species :> Post '[JSON] ()
106+ :<|> Get '[JSON] [Species])
107+
108+api :: Proxy API
109+api = Proxy
110+
111+data Species = Species
112+ { speciesName :: Text
113+ , speciesGenus :: Text
114+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
115+
116+data Genus = Genus
117+ { genusName :: Text
118+ , genusFamily :: Text
119+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
120+
121+instance FromRow Genus
122+instance FromRow Species
123+
124+server :: Connection -> Server API
125+server conn = ((\sname -> liftIO (lookupSpecies conn sname)
126+ :<|> liftIO (deleteSpecies conn sname))
127+ :<|> (\species -> liftIO $ insertSpecies conn species))
128+ :<|> (liftIO $ allSpecies conn)
129+
130+lookupSpecies :: Connection -> Text -> IO Species
131+lookupSpecies conn name = do
132+ [s] <- query conn "SELECT * FROM species WHERE species_name = ?" (Only name)
133+ return s
134+
135+deleteSpecies :: Connection -> Text -> IO ()
136+deleteSpecies conn name = do
137+ _ <- execute conn "DELETE FROM species WHERE species_name = ?" (Only name)
138+ return ()
139+
140+insertSpecies :: Connection -> Species -> IO ()
141+insertSpecies conn Species{..} = do
142+ _ <- execute conn "INSERT INTO species (species_name, species_genus) VALUES (?)" (speciesName, speciesGenus)
143+ return ()
144+
145+allSpecies :: Connection -> IO [Species]
146+allSpecies conn = do
147+ query_ conn "SELECT * FROM species"
148+
149+main :: IO ()
150+main = do
151+ conn <- connectPostgreSQL "dbname=servant-quickcheck"
152+ run 8090 (serve api $ server conn)
153+:
154+
155+(You'll also also need to run:
156+
157+```
158+createdb servant-quickcheck
159+psql --file schema.sql -d servant-quickcheck
160+```
161+
162+If you want to run this example.)
163+
164+This is a plausible effort. You might want to spend a moment thinking about what
165+could be improved.
166+
167+Here are some `servant-quickcheck`-based tests for this API:
168+
169+:d Spec.hs
170+
171+{-# LANGUAGE OverloadedStrings #-}
172+module Spec (main) where
173+
174+import Main (server, api, Species(..))
175+import Test.Hspec
176+import Test.QuickCheck.Instances
177+import Servant.QuickCheck
178+import Test.QuickCheck (Arbitrary(..))
179+import Database.PostgreSQL.Simple (connectPostgreSQL)
180+
181+spec :: Spec
182+spec = describe "the species application" $ beforeAll check $ do
183+ let pserver = do
184+ conn <- connectPostgreSQL "dbname=servant-quickcheck"
185+ return $ server conn
186+
187+
188+ it "should not return 500s" $ do
189+ withServantServer api pserver $ \url ->
190+ serverSatisfies api url defaultArgs (not500 <%> mempty)
191+
192+ it "should not return top-level json" $ do
193+ withServantServer api pserver $ \url ->
194+ serverSatisfies api url defaultArgs (onlyJsonObjects <%> mempty)
195+
196+ it "should return valid locations for 201" $ do
197+ withServantServer api pserver $ \url ->
198+ serverSatisfies api url defaultArgs (createContainsValidLocation <%> mempty)
199+
200+ it "honours Accept header" $ do
201+ withServantServer api pserver $ \url ->
202+ serverSatisfies api url defaultArgs (honoursAcceptHeader <%> mempty)
203+
204+ it "405s contain Allow header" $ do
205+ withServantServer api pserver $ \url ->
206+ serverSatisfies api url defaultArgs (notAllowedContainsValidAllow <%> mempty)
207+
208+ it "should contain WWW-Authenticate header when returning 401s" $ do
209+ withServantServer api pserver $ \url ->
210+ serverSatisfies api url defaultArgs (unauthorizedContainsWWWAuthenticate <%> mempty)
211+
212+ it "GETs should have Cache-Control header" $ do
213+ withServantServer api pserver $ \url ->
214+ serverSatisfies api url defaultArgs (getsHaveCacheControlHeader <%> mempty)
215+
216+ it "HEADs should have Cache-Control header" $ do
217+ withServantServer api pserver $ \url ->
218+ serverSatisfies api url defaultArgs (headsHaveCacheControlHeader <%> mempty)
219+
220+main :: IO ()
221+main = do
222+ hspec spec
223+
224+instance Arbitrary Species where
225+ arbitrary = Species <$> arbitrary <*> arbitrary
226+:
227+
228+
229+I won't go through all of the failures; as this is a literate haskell file, you
230+can go through them yourself if you're interested. But worth mentioning:
231+
232+ - The possible pattern match failure in `lookupSpecies` is caught.
233+ - Returning a top-level list is caught.
234+ - Not having a link for the `PostCreated` is caught.
235+
236+This last test failure illustrates an important point.
237+
238+
239+This was an example created with the knowledge of what it was supposed to
240+exemplify. To try to get a more accurate assessment of the practical usefulness
241+of `servant-quickcheck`, I tried running `serverSatisfies` with a few
242+predicates over some of the open-source `servant` servers I could find, and
243+results were also promising.
244+
245+There are probably a lot of other interesting properties that one might to add
246+besides those I've included. As an example, we could have a property that
247+all HTML is checked against, which is sometimes tricky for HTML that's
248+generated dynamically. Or check that every page has a Portuguese translation.
249+
250+### Why best practices are good
251+
252+As a side note: you might have wondered "why bother with API best practices?".
253+It is, it has to be said, a lot of extra (as in not only getting the feature done)
254+work to do, for dubious benefit. And indeed, the relevance of discoverability, for
255+example, unclear, since not that many tools use it as perhaps was anticipated.
256+
257+But `servant-quickcheck` both makes it *easier* to conform to best practices,
258+and exemplifies their advantage in enabling better tooling. If we pick 201 (Success, the 'resource' was
259+created), rather than the more generic 200 (Success), and do a *little* more work
260+by knowing to make this decision, `servant-quickcheck` knows this means there
261+should be some representation of the resource created. So it knows to ask you
262+for a link to it (the RFC creators thought to ask for this). And if you do (again,
263+a little more work), `servant-quickcheck` will know to try to look at that
264+resource by following the link, checking that it's not broken, and maybe even
265+returns a response that equivalent to the original POST request). And then it
266+finds a real bug - your application allows species with '/' in their name to
267+be created, but not queried with a 'GET' for! This, I think, is already a win.
268+
269+
270+## `serversEqual`
271+
272+There's another very appealing application of the ability to generate "sensible"
273+arbitrary requests. It's for testing that two applications are equal. We can generate arbitrary
274+requests, send them to both servers (in the same order), and check that the responses
275+are equivalent. (This was, incidentally, one of the first applications of
276+`servant-client`, albeit in a much more manual way, when we rewrote a microservice
277+originally in Python in Haskell.) Generally with rewrites, even if there's some
278+behaviour that isn't optimal, perhaps a lot of things already depend on that service
279+and make interace poorly with "improvements", so it makes sense to first mimick
280+*exactly* the original behaviour, and only then aim for improvements.
281+
282+`servant-quickcheck` provides a single function, `serversEqual`, that attempts
283+to verify the equivalence of servers. Since some aspects of responses might not
284+be relevant (for example, whether the the `Server` header is the same, or whether
285+two JSON responses have the same formatting), it allows you to provide a custom
286+equivalence function. Other than that, you need only provide an API type and two
287+URLs for testing, and the rest `serversEqual` handles.
288+
289+## Future directions: benchmarking
290+
291+What else could benefit from tooling that can automatically generate sensible
292+(*vis-a-vis* a particular application's expectations) requests?
293+
294+One area is extensive automatic benchmarking. Currently we use tools such as
295+`ab`, `wrk`, `httperf` in a very manual way - we pick a particular request that
296+we are interested in, and write a request that gets made thousands of times.
297+But now we can have a multiplicity of requests to benchmark with! This allows
298+*finding* slow endpoints, as well as (I would imagine, though I haven't actually
299+tried this yet) finding synchronization issues that make threads wait for too
300+long (such as waiting on an MVar that's not really needed), bad asymptotics
301+with respect to some other type of request.
302+
303+(On this last point, imagine not having an index in a database for "people",
304+ and having a tool that discovers that the latency on a search by first name
305+ grows linearly with the number of POST requests to a *different* endpoint! We'd
306+ need to do some work to do this well, possibly involving some machine
307+ learning, but it's an interesting and probably useful idea.)
308+
309+
310+# Conclusion
311+
312+I hope this library presents some useful functionality already, but I hope
313+you'll also think how it could be improved!
314+
315+There'll be a few more packages in the comings weeks - check back soon!
316+
317+**Note**: This post is an anansi literate file that generates multiple source
318+files. They are:
319+
320+:f Main.hs
321+|Main.hs|
322+:
323+
324+:f schema.sql
325+|schema.sql|
326+:
327+
328+:f Spec.hs
329+|Spec.hs|
330+:
doc/posts/LICENSEnew file+30
@@ -0,0 +1,30 @@
1+Copyright (c) 2016, Julian K. Arni
2+
3+All rights reserved.
4+
5+Redistribution and use in source and binary forms, with or without
6+modification, are permitted provided that the following conditions are met:
7+
8+ * Redistributions of source code must retain the above copyright
9+ notice, this list of conditions and the following disclaimer.
10+
11+ * Redistributions in binary form must reproduce the above
12+ copyright notice, this list of conditions and the following
13+ disclaimer in the documentation and/or other materials provided
14+ with the distribution.
15+
16+ * Neither the name of Julian K. Arni nor the names of other
17+ contributors may be used to endorse or promote products derived
18+ from this software without specific prior written permission.
19+
20+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
doc/posts/Makefilenew file+25
@@ -0,0 +1,25 @@
1+FILES = src/Main.hs src/Spec.hs src/schema.sql
2+
3+src/$(FILES): Announcement.anansi
4+ anansi tangle -o "src" Announcement.anansi
5+
6+announcement.md: Announcement.anansi
7+ anansi weave -o "announcement.tmp" Announcement.anansi
8+ cat announcement.tmp | tr -d '«' | tr -d '»' > announcement.md
9+ rm announcement.tmp
10+
11+.stack-work/bin/posts: $(FILES) stack.yaml posts.cabal
12+ stack build
13+
14+announcement.html: announcement.md
15+ pandoc announcement.md -t html > announcement.html
16+
17+run: .stack-work/bin/posts
18+ stack exec posts
19+
20+test: .stack-work/bin/posts
21+ stack test
22+
23+post: announcement.html
24+
25+.PHONY: post run test
doc/posts/Setup.hsnew file+2
@@ -0,0 +1,2 @@
1+import Distribution.Simple
2+main = defaultMain
doc/posts/announcement.mdnew file+318
@@ -0,0 +1,318 @@
1+
2+# Announcing servant-quickcheck
3+
4+Some time ago, we released `servant-mock`. The idea behind it is to use
5+`QuickCheck` to create a mock server that accords with a servant API. Not long
6+after, we started thinking about an analog that would, instead of mocking a
7+server, mock a client instead - i.e., generate random requests that conform to
8+an API description.
9+
10+This is much closer to the traditional use of `QuickCheck`. The most obvious
11+use-case is checking that properties hold of an *entire* server rather than of
12+individual endpoints. (But there are other uses that you can skip to if they
13+sound more interesting.)
14+
15+## `serverSatisfies`
16+
17+A useful guideline when writing and maintaing software is that, if there isn't
18+a test for a behaviour or property, sooner or later that property will be broken.
19+Another important perspective is that tests are a form of documentation - the
20+present developer telling future ones "this matters, and should be this way".
21+
22+The advantage of using tests for this form of documentation is that there's
23+simply too much information to convey, some of it only relevant to very specific
24+use cases, and rather than overload developers with an inexhaustible quantity of
25+details that would be hard to keep track of or remember, tests are a mechanism
26+of reminding developers of *only the relevant information, at the right time*.
27+<<EXAMPLE>>.
28+
29+We might hope that we could use tests to communicate the wide array of best
30+practices that have developed around APIs. About to return a top-level integer
31+in JSON? A test should say that's bad practice. About to not catch exceptions
32+and give a more meaningful HTTP status code? Another test there to stop you.
33+
34+Traditionally, in web services these things get done at the level of *individual*
35+endpoints. But this means that if a developer who hasn't had extensive experience with web
36+programming best practices writes a *new* endpoint which *does* return a top-level
37+integer literal, there's no test there to stop her. Code review might help, but
38+code review is much more error prone than tests, and really only meant for those
39+things that are too subtle to automate. (Indeed, if code review were such a reliable
40+defense mechanism against bugs and bad code, why have tests and linters at all?)
41+
42+The problem, then, with thinking about tests as only existing at the level of individual
43+endpoints is that there are no tests *for* tests - tests that check that new
44+behaviour and tests conforms to higher-level, more general best practices.
45+
46+`servant-quickcheck` aims to solve that. It allows describing properties that
47+*all* endpoints must satisfy. If a new endpoint comes along, it too will be
48+tested for that property, without any further work.
49+
50+Why isn't this idea already popular? Well, most web frameworks don't have a
51+reified description of APIs (beyond perhaps the routes). When you don't know
52+what the endpoints of an application are, and what request body they expect,
53+trying to generate arbitrary requests is almost entirely going to result in
54+404s (not found) and 400s (bad request). Maybe one in a thousand requests will
55+actually test a handler. Not very useful.
56+
57+`servant` applications, on the other hand, have a machine-readable API description
58+already available. And they already associate "correct" requests with particular
59+types. It's a small step, therefore, to generate 'arbitrary' values for these
60+requests, and all of them will go through to your handlers. (Note: all of the
61+uses of `servant-quickcheck` work with applications *not* written with servant-server -
62+and indeed not *in Haskell - but the API must be described with the servant
63+DSL.)
64+
65+Let's see how this works in practice. As a running example, let's use a simple
66+service that allows adding, removing, and querying biological species. Our SQL
67+schema is:
68+
69+
70+> **schema.sql**
71+
72+>
73+> CREATE TABLE genus (
74+> genus_name text PRIMARY KEY,
75+> genus_family text NOT NULL
76+> );
77+>
78+> CREATE TABLE species (
79+> species_name text PRIMARY KEY,
80+> species_genus text NOT NULL REFERENCES genus (genus_name)
81+> )
82+
83+
84+And our actual application:
85+
86+
87+> **Main.hs**
88+
89+> {-# LANGUAGE DataKinds #-}
90+> {-# LANGUAGE DeriveAnyClass #-}
91+> {-# LANGUAGE DeriveGeneric #-}
92+> {-# LANGUAGE TypeOperators #-}
93+> {-# LANGUAGE OverloadedStrings #-}
94+> {-# LANGUAGE RecordWildCards #-}
95+> module Main where
96+>
97+> import Servant
98+> import Data.Aeson
99+> import Database.PostgreSQL.Simple
100+> import GHC.Generics (Generic)
101+> import Data.Text (Text)
102+> import Network.Wai.Handler.Warp
103+> import Control.Monad.IO.Class (liftIO)
104+>
105+> type API
106+> = "species" :> (Capture "species-name" Text :> ( Get '[JSON] Species
107+> :<|> Delete '[JSON] ())
108+> :<|> ReqBody '[JSON] Species :> Post '[JSON] ())
109+> -- The plural of 'species' is unfortunately also 'species'
110+> :<|> "speciess" :> Get '[JSON] [Species]
111+>
112+> api :: Proxy API
113+> api = Proxy
114+>
115+> data Species = Species
116+> { speciesName :: Text
117+> , speciesGenus :: Text
118+> } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
119+>
120+> data Genus = Genus
121+> { genusName :: Text
122+> , genusFamily :: Text
123+> } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
124+>
125+> instance FromRow Genus
126+> instance FromRow Species
127+>
128+> server :: Connection -> Server API
129+> server conn = ((\sname -> liftIO (lookupSpecies conn sname)
130+> :<|> liftIO (deleteSpecies conn sname))
131+> :<|> (\species -> liftIO $ insertSpecies conn species))
132+> :<|> (liftIO $ allSpecies conn)
133+>
134+> lookupSpecies :: Connection -> Text -> IO Species
135+> lookupSpecies conn name = do
136+> [s] <- query conn "SELECT * FROM species WHERE species_name = ?" (Only name)
137+> return s
138+>
139+> deleteSpecies :: Connection -> Text -> IO ()
140+> deleteSpecies conn name = do
141+> _ <- execute conn "DELETE FROM species WHERE species_name = ?" (Only name)
142+> return ()
143+>
144+> insertSpecies :: Connection -> Species -> IO ()
145+> insertSpecies conn Species{..} = do
146+> _ <- execute conn "INSERT INTO species (species_name, species_genus) VALUES (?)" (speciesName, speciesGenus)
147+> return ()
148+>
149+> allSpecies :: Connection -> IO [Species]
150+> allSpecies conn = do
151+> query_ conn "SELECT * FROM species"
152+>
153+> main :: IO ()
154+> main = do
155+> conn <- connectPostgreSQL "dbname=servant-quickcheck"
156+> run 8090 (serve api $ server conn)
157+
158+
159+(You'll also also need to run:
160+
161+```
162+createdb servant-quickcheck
163+psql --file schema.sql -d servant-quickcheck
164+```
165+
166+If you want to run this example.)
167+
168+This is a plausible effort. You might want to spend a moment thinking about what
169+could be improved.
170+
171+
172+> **Spec.hs**
173+
174+>
175+> {-# LANGUAGE OverloadedStrings #-}
176+> module Spec (main) where
177+>
178+> import Main (server, api, Species(..))
179+> import Test.Hspec
180+> import Test.QuickCheck.Instances
181+> import Servant.QuickCheck
182+> import Test.QuickCheck (Arbitrary(..))
183+> import Database.PostgreSQL.Simple (connectPostgreSQL)
184+>
185+> spec :: Spec
186+> spec = describe "the species application" $ beforeAll check $ do
187+> let pserver = do
188+> conn <- connectPostgreSQL "dbname=servant-quickcheck"
189+> return $ server conn
190+>
191+>
192+> it "should not return 500s" $ do
193+>
194+> it "should not return 500s" $ do
195+> withServantServer api pserver $ \url ->
196+> serverSatisfies api url defaultArgs (not500 <%> mempty)
197+>
198+> it "should not return top-level json" $ do
199+> withServantServer api pserver $ \url ->
200+> serverSatisfies api url defaultArgs (onlyJsonObjects <%> mempty)
201+>
202+> where
203+> check = do
204+> mvar <- newMVar []
205+> withServantServer api pserver $ \url ->
206+> serverSatisfies api url defaultArgs (onlyJsonObjects <%> mempty)
207+>
208+> main :: IO ()
209+> main = do
210+> hspec spec
211+>
212+> instance Arbitrary Species where
213+> arbitrary = Species <$> arbitrary <*> arbitrary
214+
215+
216+But this fails in quite a few ways.
217+
218+
219+<<TODO>>
220+
221+This was an example created with the knowledge of what it was supposed to
222+exemplify. To try to get a more accurate assessment of the practical usefulness
223+of `servant-quickcheck`, I tried running `serverSatisfies` with a few
224+predicates over some of the open-source `servant` servers I could find, and
225+results were also promising.
226+
227+There are probably a lot of other interesting properties that one might to add
228+besides those I've included. As an example, we could have a property that
229+all HTML is checked against, which is sometimes tricky for HTML that's
230+generated dynamically. Or check that every page has a Portuguese translation.
231+
232+### Why best practices are good
233+
234+As a side note: you might have wondered "why bother with API best practices?".
235+It is, it has to be said, a lot of extra (as in not only getting the feature done)
236+work to do, for dubious benefit. And indeed, the relevance of discoverability, for
237+example, unclear, since not that many tools use it as perhaps was anticipated.
238+
239+But `servant-quickcheck` both makes it *easier* to conform to best practices,
240+and exemplifies their advantage in enabling better tooling. If we pick 201 (Success, the 'resource' was
241+created), rather than the more generic 200 (Success), and do a *little* more work
242+by knowing to make this decision, `servant-quickcheck` knows this means there
243+should be some representation of the resource created. So it knows to ask you
244+for a link to it (the RFC creators thought to ask for this). And if you do (again,
245+a little more work), `servant-quickcheck` will know to try to look at that
246+resource by following the link, checking that it's not broken, and maybe even
247+returns a response that equivalent to the original POST request). And then it
248+finds a real bug - your application allows species with '/' in their name to
249+be created, but not queried with a 'GET' for! This, I think, is already a win.
250+
251+
252+## `serversEqual`
253+
254+There's another very appealing application of the ability to generate "sensible"
255+arbitrary requests. It's for testing that two applications are equal. We can generate arbitrary
256+requests, send them to both servers (in the same order), and check that the responses
257+are equivalent. (This was, incidentally, one of the first applications of
258+`servant-client`, albeit in a much more manual way, when we rewrote a microservice
259+originally in Python in Haskell.) Generally with rewrites, even if there's some
260+behaviour that isn't optimal, perhaps a lot of things already depend on that service
261+and make interace poorly with "improvements", so it makes sense to first mimick
262+*exactly* the original behaviour, and only then aim for improvements.
263+
264+`servant-quickcheck` provides a single function, `serversEqual`, that attempts
265+to verify the equivalence of servers. Since some aspects of responses might not
266+be relevant (for example, whether the the `Server` header is the same, or whether
267+two JSON responses have the same formatting), it allows you to provide a custom
268+equivalence function. Other than that, you need only provide an API type and two
269+URLs for testing, and the rest `serversEqual` handles.
270+
271+## Future directions: benchmarking
272+
273+What else could benefit from tooling that can automatically generate sensible
274+(*vis-a-vis* a particular application's expectations) requests?
275+
276+One area is extensive automatic benchmarking. Currently we use tools such as
277+`ab`, `wrk`, `httperf` in a very manual way - we pick a particular request that
278+we are interested in, and write a request that gets made thousands of times.
279+But now we can have a multiplicity of requests to benchmark with! This allows
280+*finding* slow endpoints, as well as (I would imagine, though I haven't actually
281+tried this yet) finding synchronization issues that make threads wait for too
282+long (such as waiting on an MVar that's not really needed), bad asymptotics
283+with respect to some other type of request.
284+
285+(On this last point, imagine not having an index in a database for "people",
286+ and having a tool that discovers that the latency on a search by first name
287+ grows linearly with the number of POST requests to a *different* endpoint! We'd
288+ need to do some work to do this well, possibly involving some machine
289+ learning, but it's an interesting and probably useful idea.)
290+
291+
292+# Conclusion
293+
294+I hope this library presents some useful functionality already, but I hope
295+you'll also think how it could be improved!
296+
297+There'll be a few more packages in the comings weeks - check back soon!
298+
299+**Note**: This post is an anansi literate file that generates multiple source
300+files. They are:
301+
302+
303+> ** Main.hs**
304+
305+> Main.hs
306+
307+
308+
309+> ** schema.sql**
310+
311+> schema.sql
312+
313+
314+
315+> ** Spec.hs**
316+
317+> Spec.hs
318+
doc/posts/posts.cabalnew file+47
@@ -0,0 +1,47 @@
1+-- Initial posts.cabal generated by cabal init. For further documentation,
2+-- see http://haskell.org/cabal/users-guide/
3+
4+name: posts
5+version: 0.1.0.0
6+-- synopsis:
7+-- description:
8+license: BSD3
9+license-file: LICENSE
10+author: Julian K. Arni
11+maintainer: jkarni@gmail.com
12+-- copyright:
13+-- category:
14+build-type: Simple
15+-- extra-source-files:
16+cabal-version: >=1.10
17+
18+executable posts
19+ main-is: Main.hs
20+ build-depends: base >=4.8 && <4.9
21+ , servant-server >=0.5 && <0.8
22+ , aeson >=0.9 && <0.12
23+ , text == 1.*
24+ , warp >=3.0 && <3.3
25+ , transformers >=0.4 && <0.5
26+ , postgresql-simple
27+ hs-source-dirs: src
28+ ghc-options: -Wall -O2 -threaded
29+ default-language: Haskell2010
30+
31+test-suite spec
32+ type: exitcode-stdio-1.0
33+ ghc-options: -Wall -O2 -threaded
34+ default-language: Haskell2010
35+ hs-source-dirs: src
36+ main-is: Spec.hs
37+ build-depends: base == 4.*
38+ , servant-quickcheck
39+ , hspec == 2.*
40+ , servant-server >=0.5 && <0.8
41+ , aeson >=0.9 && <0.12
42+ , text == 1.*
43+ , warp >=3.0 && <3.3
44+ , transformers >=0.4 && <0.5
45+ , postgresql-simple
46+ , quickcheck-instances
47+ , QuickCheck
doc/posts/src/Main.hsnew file+71
@@ -0,0 +1,71 @@
1+#line 296 "Announcement.anansi"
2+
3+#line 86 "Announcement.anansi"
4+{-# LANGUAGE DataKinds #-}
5+{-# LANGUAGE DeriveAnyClass #-}
6+{-# LANGUAGE DeriveGeneric #-}
7+{-# LANGUAGE TypeOperators #-}
8+{-# LANGUAGE OverloadedStrings #-}
9+{-# LANGUAGE RecordWildCards #-}
10+module Main where
11+
12+import Servant
13+import Data.Aeson
14+import Database.PostgreSQL.Simple
15+import GHC.Generics (Generic)
16+import Data.Text (Text)
17+import Network.Wai.Handler.Warp
18+import Control.Monad.IO.Class (liftIO)
19+
20+type API
21+ = "species" :> (Capture "species-name" Text :> ( Get '[JSON] Species
22+ :<|> Delete '[JSON] ())
23+ :<|> ReqBody '[JSON] Species :> Post '[JSON] ())
24+ -- The plural of 'species' is unfortunately also 'species'
25+ :<|> "speciess" :> Get '[JSON] [Species]
26+
27+api :: Proxy API
28+api = Proxy
29+
30+data Species = Species
31+ { speciesName :: Text
32+ , speciesGenus :: Text
33+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
34+
35+data Genus = Genus
36+ { genusName :: Text
37+ , genusFamily :: Text
38+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
39+
40+instance FromRow Genus
41+instance FromRow Species
42+
43+server :: Connection -> Server API
44+server conn = ((\sname -> liftIO (lookupSpecies conn sname)
45+ :<|> liftIO (deleteSpecies conn sname))
46+ :<|> (\species -> liftIO $ insertSpecies conn species))
47+ :<|> (liftIO $ allSpecies conn)
48+
49+lookupSpecies :: Connection -> Text -> IO Species
50+lookupSpecies conn name = do
51+ [s] <- query conn "SELECT * FROM species WHERE species_name = ?" (Only name)
52+ return s
53+
54+deleteSpecies :: Connection -> Text -> IO ()
55+deleteSpecies conn name = do
56+ _ <- execute conn "DELETE FROM species WHERE species_name = ?" (Only name)
57+ return ()
58+
59+insertSpecies :: Connection -> Species -> IO ()
60+insertSpecies conn Species{..} = do
61+ _ <- execute conn "INSERT INTO species (species_name, species_genus) VALUES (?)" (speciesName, speciesGenus)
62+ return ()
63+
64+allSpecies :: Connection -> IO [Species]
65+allSpecies conn = do
66+ query_ conn "SELECT * FROM species"
67+
68+main :: IO ()
69+main = do
70+ conn <- connectPostgreSQL "dbname=servant-quickcheck"
71+ run 8090 (serve api $ server conn)
doc/posts/src/Spec.hsnew file+40
@@ -0,0 +1,40 @@
1+#line 304 "Announcement.anansi"
2+
3+#line 171 "Announcement.anansi"
4+
5+{-# LANGUAGE OverloadedStrings #-}
6+module Spec (main) where
7+
8+import Main (server, api, Species(..))
9+import Test.Hspec
10+import Test.QuickCheck.Instances
11+import Servant.QuickCheck
12+import Test.QuickCheck (Arbitrary(..))
13+import Database.PostgreSQL.Simple (connectPostgreSQL)
14+
15+spec :: Spec
16+spec = describe "the species application" $ beforeAll check $ do
17+ let pserver = do
18+ conn <- connectPostgreSQL "dbname=servant-quickcheck"
19+ return $ server conn
20+
21+
22+ it "should not return 500s" $ do
23+ withServantServer api pserver $ \url ->
24+ serverSatisfies api url defaultArgs (not500 <%> mempty)
25+
26+ it "should not return top-level json" $ do
27+ withServantServer api pserver $ \url ->
28+ serverSatisfies api url defaultArgs (onlyJsonObjects <%> mempty)
29+
30+ it "should return valid locations for 201" $ do
31+ withServantServer api pserver $ \url ->
32+ serverSatisfies api url defaultArgs (createContainsValidLocation <%> mempty)
33+
34+
35+main :: IO ()
36+main = do
37+ hspec spec
38+
39+instance Arbitrary Species where
40+ arbitrary = Species <$> arbitrary <*> arbitrary
doc/posts/src/schema.sqlnew file+9
@@ -0,0 +1,9 @@
1+CREATE TABLE genus (
2+ genus_name text PRIMARY KEY,
3+ genus_family text NOT NULL
4+);
5+
6+CREATE TABLE species (
7+ species_name text PRIMARY KEY,
8+ species_genus text NOT NULL REFERENCES genus (genus_name)
9+)
doc/posts/stack.yamlnew file+46
@@ -0,0 +1,46 @@
1+# This file was automatically generated by stack init
2+# For more information, see: http://docs.haskellstack.org/en/stable/yaml_configuration/
3+
4+
5+# Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2)
6+resolver: lts-5.14
7+
8+# Local packages, usually specified by relative directory name
9+packages:
10+- '../..'
11+- '.'
12+# Packages to be pulled from upstream that are not in the resolver (e.g., acme-missiles-0.3)
13+extra-deps:
14+- aeson-0.11.1.4
15+- fast-logger-2.4.6
16+- http2-1.6.0
17+- servant-0.7
18+- servant-server-0.7
19+- servant-client-0.7
20+- servant-quickcheck-0.1.0.0
21+- text-1.2.2.1
22+- warp-3.2.6
23+
24+# Override default flag values for local packages and extra-deps
25+flags: {}
26+
27+# Extra package databases containing global packages
28+extra-package-dbs: []
29+
30+# Control whether we use the GHC we find on the path
31+# system-ghc: true
32+
33+# Require a specific version of stack, using version ranges
34+# require-stack-version: -any # Default
35+# require-stack-version: >= 1.0.0
36+
37+# Override the architecture used by stack, especially useful on Windows
38+# arch: i386
39+# arch: x86_64
40+
41+# Extra directories used by stack for building
42+# extra-include-dirs: [/path/to/dir]
43+# extra-lib-dirs: [/path/to/dir]
44+
45+# Allow a newer minor version of GHC than the snapshot specifies
46+# compiler-check: newer-minor
example/Main.hsnew file+31
@@ -0,0 +1,31 @@
1+{-# LANGUAGE OverloadedStrings, DataKinds #-}
2+module Main (main) where
3+
4+import Servant
5+import Servant.QuickCheck
6+import Test.Hspec
7+import Data.Text (Text)
8+import System.Environment (getArgs)
9+
10+main :: IO ()
11+main = do
12+ args <- getArgs
13+ case args of
14+ [] -> putStrLn "Not running without arguments. Try --help or --fail-fast."
15+ _ -> hspec spec
16+
17+-- Change to String to reproduce
18+-- https://github.com/haskell-servant/servant-quickcheck/issues/41
19+type API = Get '[PlainText] Text
20+
21+api :: Proxy API
22+api = Proxy
23+
24+server :: Server API
25+server = return "Sigurð Fáfnirslayer"
26+
27+spec :: Spec
28+spec = describe "example server" $
29+ it "mangles UTF-8 in error messages" $
30+ withServantServer api (return server) $ \burl ->
31+ serverSatisfies api burl defaultArgs (getsHaveCacheControlHeader <%> mempty)
servant-quickcheck.cabal+126−76
@@ -1,88 +1,138 @@
1−name: servant-quickcheck
2−version: 0.1.0.0
3−synopsis: QuickCheck entire APIs
1+name: servant-quickcheck
2+version: 0.0.9.1
3+synopsis: QuickCheck entire APIs
44description:
5− This packages provides QuickCheck properties that are tested across an entire
6− API.
5+ This packages provides QuickCheck properties that are tested across an entire
6+ API.
77
8−license: BSD3
9−license-file: LICENSE
10−author: Julian K. Arni
11−maintainer: jkarni@gmail.com
12−category: Web
13−build-type: Simple
14−cabal-version: >=1.10
8+license: BSD3
9+license-file: LICENSE
10+author: Julian K. Arni
11+maintainer: jkarni@gmail.com
12+category: Web
13+build-type: Simple
14+cabal-version: >=1.10
15+extra-source-files: CHANGELOG.yaml
16+tested-with: GHC ==8.0.2 || ==8.2.2 || ==8.4.4 || ==8.6.5 || == 8.8.3
17+
18+source-repository head
19+ type: git
20+ location: https://github.com/haskell-servant/servant-quickcheck
1521
1622flag long-tests
1723 description: Run more QuickCheck tests
1824 default: False
1925
2026library
21− exposed-modules: Servant.QuickCheck
22− , Servant.QuickCheck.Internal
23− , Servant.QuickCheck.Internal.Benchmarking
24− , Servant.QuickCheck.Internal.Predicates
25− , Servant.QuickCheck.Internal.Testable
26− , Servant.QuickCheck.Internal.QuickCheck
27− build-depends: base >=4.8 && <4.9
28− , QuickCheck == 2.8.*
29− , bytestring == 0.10.*
30− , aeson > 0.10 && < 0.12
31− , mtl == 2.2.*
32− , http-client == 0.4.*
33− , http-types == 0.9.*
34− , servant-client == 0.7.*
35− , servant-server == 0.7.*
36− , servant == 0.7.*
37− , warp >= 3.2.4 && < 3.3
38− , process == 1.2.*
39− , temporary == 1.2.*
40− , hspec
41− hs-source-dirs: src
42− default-extensions: TypeOperators
43− , FlexibleInstances
44− , FlexibleContexts
45− , DataKinds
46− , GADTs
47− , MultiParamTypeClasses
48− , DeriveFunctor
49− , RankNTypes
50− , ConstraintKinds
51− , DeriveGeneric
52− default-language: Haskell2010
27+ exposed-modules:
28+ Servant.QuickCheck
29+ Servant.QuickCheck.Internal
30+ Servant.QuickCheck.Internal.Equality
31+ Servant.QuickCheck.Internal.ErrorTypes
32+ Servant.QuickCheck.Internal.HasGenRequest
33+ Servant.QuickCheck.Internal.Predicates
34+ Servant.QuickCheck.Internal.QuickCheck
35+
36+ build-depends:
37+ aeson >=0.8 && <2
38+ , base >=4.9 && <4.15
39+ , base-compat-batteries >=0.10.1 && <0.12
40+ , bytestring >=0.10 && <0.11
41+ , case-insensitive >=1.2 && <1.3
42+ , clock >=0.7 && <0.9
43+ , data-default-class >=0.0 && <0.2
44+ , hspec >=2.5.6 && <2.8
45+ , http-client >=0.4.30 && <0.8
46+ , http-media >=0.6 && <0.9
47+ , http-types >=0.8 && <0.13
48+ , mtl >=2.1 && <2.3
49+ , pretty >=1.1 && <1.2
50+ , process >=1.2 && <1.7
51+ , QuickCheck >=2.7 && <2.15
52+ , servant >=0.17 && <0.19
53+ , servant-client >=0.17 && <0.19
54+ , servant-server >=0.17 && <0.19
55+ , split >=0.2 && <0.3
56+ , string-conversions >=0.3 && <0.5
57+ , temporary >=1.2 && <1.4
58+ , text >=1 && <2
59+ , time >=1.5 && <1.11
60+ , warp >=3.2.4 && <3.4
61+
62+ if !impl(ghc >=8.0)
63+ build-depends: semigroups >=0.18.3 && <0.20
64+
65+ hs-source-dirs: src
66+ default-extensions:
67+ NoImplicitPrelude
68+ ConstraintKinds
69+ DataKinds
70+ DeriveDataTypeable
71+ DeriveFunctor
72+ DeriveGeneric
73+ FlexibleContexts
74+ FlexibleInstances
75+ FunctionalDependencies
76+ GADTs
77+ KindSignatures
78+ MultiParamTypeClasses
79+ OverloadedStrings
80+ RankNTypes
81+ ScopedTypeVariables
82+ TypeOperators
83+
84+ default-language: Haskell2010
5385
5486test-suite spec
55− type: exitcode-stdio-1.0
56− ghc-options: -Wall -O2 -threaded
57− default-language: Haskell2010
58− hs-source-dirs: test
59− main-is: Spec.hs
60− other-modules: Servant.QuickCheck.InternalSpec
61− build-depends: base == 4.*
62− , servant-quickcheck
63− , hspec
64− , http-client
65− , warp
66− , servant-server
67− , servant-client
68− , transformers
69− , QuickCheck
70− default-extensions: TypeOperators
71− , FlexibleInstances
72− , FlexibleContexts
73− , DataKinds
87+ type: exitcode-stdio-1.0
88+ ghc-options: -Wall -threaded
89+ default-language: Haskell2010
90+ hs-source-dirs: test
91+ main-is: Spec.hs
92+ other-modules: Servant.QuickCheck.InternalSpec
93+ build-tool-depends: hspec-discover:hspec-discover -any
94+ build-depends:
95+ aeson
96+ , base
97+ , base-compat-batteries
98+ , blaze-html
99+ , bytestring
100+ , hspec
101+ , hspec-core >=2.5.5 && <2.8
102+ , http-client
103+ , QuickCheck
104+ , quickcheck-io
105+ , servant
106+ , servant-blaze
107+ , servant-client
108+ , servant-quickcheck
109+ , servant-server
110+ , transformers
111+ , warp
112+
113+ default-extensions:
114+ NoImplicitPrelude
115+ DataKinds
116+ FlexibleContexts
117+ FlexibleInstances
118+ GADTs
119+ OverloadedStrings
120+ ScopedTypeVariables
121+ TypeOperators
122+
74123 if flag(long-tests)
75− cpp-options: -DLONG_TESTS
124+ cpp-options: -DLONG_TESTS
125+
126+test-suite example
127+ type: exitcode-stdio-1.0
128+ main-is: Main.hs
129+ hs-source-dirs: example
130+ ghc-options: -Wall
131+ build-depends:
132+ base
133+ , hspec
134+ , servant-quickcheck
135+ , servant-server
136+ , text
76137
77−-- test-suite doctests
78−-- default-language: Haskell2010
79−-- type: exitcode-stdio-1.0
80−-- ghc-options: -threaded
81−-- main-is: Doctest.hs
82−-- hs-source-dirs: test
83−-- build-depends: base >4 && <5
84−-- , doctest
85−-- , filemanip
86−-- , directory
87−-- , filepath
88−-- HS-Source-Dirs: test
138+ default-language: Haskell2010
src/Servant/QuickCheck.hs+63−69
@@ -6,89 +6,83 @@
66-- tested itself need not be implemented with @servant-server@ (or indeed,
77-- written in Haskell).
88--
9−-- /N.B./ The examples given here assume the following setup:
10−--
11−-- > import Servant
12−-- > import Servant.QuickCheck
13−-- > import Test.Hspec
14−-- >
15−-- > type API = ReqBody '[JSON] Int :> Post '[JSON] String
16−-- >
17−-- > api :: Proxy API
18−-- > api = Proxy
9+-- The documentation of the <#useful-preds Useful predicates> sections is
10+-- meant to serve as a set of helpful pointers for learning more about best
11+-- practices concerning REST APIs.
1912module Servant.QuickCheck
2013 (
2114
22− -- * Server properties
23− -- | Functions to verify that a server meets certain properties.
24− --
25− -- Example:
26− --
27− -- > server :: Server API
28− -- > server = return . show
29− -- >
30− -- >
31− -- > test :: Spec
32− -- > test = describe "my server" $ do
33− -- >
34− -- > it "never throws a 500 on valid input" $ do
35− -- > withServantServer api server $ \url ->
36− -- > serverSatisfiers api url emptyPredicates never500s 100
15+ -- * Property testing
3716 serverSatisfies
38−
39− -- * Server equality
40− -- | Functions to verify that two servers behave identically.
17+ -- ** Predicates
18+ -- *** #useful-preds# Useful predicates
19+ -- | The predicates below are often useful. Some check RFC compliance; some are
20+ -- best practice, and some are useful to check that APIs follow in-house
21+ -- best-practices. Included in the documentation for each is a list of
22+ -- references to any relevant RFCs and other links, as well as what type of
23+ -- predicate it is (__RFC Compliance__, __Best Practice__, __Optional__).
4124 --
42− -- This can be useful when for example rewriting or refactoring an
43− -- application.
44− --
45− -- Example:
46− --
47− -- > server :: Server API
48− -- > server = return . show
49− -- >
50− -- > server2 :: Server API
51− -- > server2 = const $ return "hi"
52− -- >
53− -- > test :: Spec
54− -- > test = describe "my new server" $ do
55− -- >
56− -- > it "behaves like the old one" $ do
57− -- > withServantServer api server $ \url1 ->
58− -- > withServantServer api server2 $ \url2 ->
59− -- > serversEqual api url1 url2 100
60− --
61− , serversEqual
25+ -- RFCs distinguish between the force of requirements (e.g. __MUST__ vs.
26+ -- __SHOULD__). __RFC Compliance__ includes any absolute requirements present
27+ -- in RFCs. The __Best Practices__ includes, in addition to RFC
28+ -- recommendations, recommendations found elsewhere or generally accepted.
29+ , not500
30+ , notLongerThan
31+ , onlyJsonObjects
32+ , honoursAcceptHeader
33+ , notAllowedContainsAllowHeader
34+ , unauthorizedContainsWWWAuthenticate
35+ , getsHaveLastModifiedHeader
36+ , getsHaveCacheControlHeader
37+ , headsHaveCacheControlHeader
38+ , createContainsValidLocation
39+ -- * Html Predicates
40+ , htmlIncludesDoctype
6241
63− -- * Server benchmarking
64− -- | Functions that randomly generate and run benchmarking scripts
65− , serverBenchmark
66− , BenchOptions(..)
67− , defaultBenchOptions
42+ -- *** Predicate utilities and types
43+ , (<%>)
44+ , Predicates
45+ , ResponsePredicate(..)
46+ , RequestPredicate(..)
6847
48+ -- * Equality testing
49+ , serversEqual
50+ -- ** Response equality
51+ -- | Often the normal equality of responses is not what we want. For example,
52+ -- if responses contain a @Date@ header with the time of the response,
53+ -- responses will fail to be equal even though they morally are. This datatype
54+ -- represents other means of checking equality
55+ -- *** Useful @ResponseEquality@s
56+ , bodyEquality
57+ , jsonEquality
58+ , allEquality
59+ -- ** Response equality type
60+ , ResponseEquality(..)
6961
7062 -- * Test setup helpers
7163 -- | Helpers to setup and teardown @servant@ servers during tests.
7264 , withServantServer
65+ , withServantServerAndContext
66+ , defaultArgs
7367
74− -- * Predicates
75− -- | Predicates (functions with signatures @a -> Bool@) are used to filter
76− -- out QuickCheck-generated values (so as to specify that requests must
77− -- possess certain properties) and to check that the response specifies the
78− -- expected properties.
79− , Predicates
80− , emptyPredicates
81− , addPredicate
82− , addPolyPredicate
83−
84− -- ** Predicate convenience functions
85− , addRightPredicate
86− , addLeftPredicate
68+ -- ** Re-exports
69+ -- | Types and constructors from other packages that are generally needed for
70+ -- using @servant-quickcheck@.
71+ , BaseUrl(..)
72+ , Scheme(..)
73+ , Args(..)
74+ , Proxy(..)
8775
88− -- ** Useful predicates
89− , never500s
90− , onlyJsonObjects
9176
9277 ) where
9378
79+import Data.Proxy (Proxy (..))
80+import Servant.Client (BaseUrl (..), Scheme (..))
9481import Servant.QuickCheck.Internal
82+import Test.QuickCheck (Args (..), stdArgs)
83+
84+-- | QuickCheck @Args@ with 1000 rather than 100 test cases.
85+--
86+-- /Since 0.0.0.0/
87+defaultArgs :: Args
88+defaultArgs = stdArgs { maxSuccess = 1000 }
src/Servant/QuickCheck/Internal.hs+5−4
@@ -1,6 +1,7 @@
11module Servant.QuickCheck.Internal (module X) where
22
3−import Servant.QuickCheck.Internal.Testable as X
4−import Servant.QuickCheck.Internal.Predicates as X
5−import Servant.QuickCheck.Internal.QuickCheck as X
6−import Servant.QuickCheck.Internal.Benchmarking as X
3+import Servant.QuickCheck.Internal.Equality as X
4+import Servant.QuickCheck.Internal.ErrorTypes as X
5+import Servant.QuickCheck.Internal.HasGenRequest as X
6+import Servant.QuickCheck.Internal.Predicates as X
7+import Servant.QuickCheck.Internal.QuickCheck as X
src/Servant/QuickCheck/Internal/Benchmarking.hsdeleted−87
@@ -1,87 +0,0 @@
1−-- | This module contains benchmark-related logic.
2−--
3−-- Currently it generates 'wrk' scripts rather than benchmarking directly with
4−-- the @servant-client@ functions since the performance of 'wrk' is
5−-- significantly better.
6−module Servant.QuickCheck.Internal.Benchmarking where
7−
8−import Data.ByteString (ByteString)
9−import Data.ByteString.Lazy (toStrict)
10−import Network.HTTP.Client
11−import Network.HTTP.Types
12−import Servant.Client
13−
14−data BenchOptions = BenchOptions
15− { duration :: Int
16− , threads :: Int
17− , connections :: Int
18− , noOfTests :: Int
19− } deriving (Eq, Show, Read)
20−
21−defaultBenchOptions :: BenchOptions
22−defaultBenchOptions = BenchOptions
23− { duration = 10
24− , threads = 1
25− , connections = 10
26− , noOfTests = 10
27− }
28−
29−data WrkScript = WrkScript
30− { wrkScheme :: Scheme
31− , wrkHost :: ByteString
32− , wrkPort :: Int
33− , wrkMethod :: Method
34− , wrkPath :: ByteString
35− , wrkHeaders :: [Header]
36− , wrkBody :: ByteString
37− } deriving (Eq, Show)
38−
39−mkScript :: WrkScript -> String
40−mkScript w
41− = "wrk.scheme = \"" ++ sscheme (wrkScheme w) ++ "\""
42− ++ "\nwrk.host = " ++ show (wrkHost w)
43− ++ "\nwrk.port = " ++ show (wrkPort w)
44− ++ "\nwrk.method = " ++ show (wrkMethod w)
45− ++ "\nwrk.path = " ++ show (wrkPath w)
46− ++ foldr (\(h,v) old -> old ++ "\nwrk.headers[" ++ show h ++ "] = " ++ show v)
47− ""
48− (wrkHeaders w)
49− ++ "\nwrk.body = " ++ show (wrkBody w)
50− ++ "\n" ++ reportFmt
51− where
52− sscheme Http = "http"
53− sscheme Https = "https"
54−
55−reqToWrk :: Request -> WrkScript
56−reqToWrk r = WrkScript
57− { wrkScheme = Http
58− , wrkHost = host r
59− , wrkPort = port r
60− , wrkMethod = method r
61− , wrkPath = path r
62− , wrkHeaders = requestHeaders r
63− , wrkBody = case requestBody r of
64− RequestBodyLBS r' -> toStrict r'
65− _ -> error "expecting RequestBodyLBS"
66− }
67−
68−reportFmt :: String
69−reportFmt
70− = "done = function(summary, latency, requests)\n"
71− ++ " for _, p in pairs({ 50, 75, 99, 99.999 }) do\n"
72− ++ " n = latency:percentile(p)\n"
73− ++ " io.write(string.format(\"%g%%, %d\\n\", p, n))\n"
74− ++ " end\n"
75− ++ "end\n"
76−
77−{-data BenchResult = BenchResult-}
78− {-{ benchReq :: Request-}
79− {-, benchLatencyDist :: [(Percentile, Microsecs)]-}
80− {-, benchLatencyAvg :: Microsecs-}
81− {-} deriving (Eq, Show, Read, Generic)-}
82−
83−{-newtype Microsecs = Microsecs { unMicroSecs :: Int }-}
84− {-deriving (Eq, Show, Read, Generic)-}
85−
86−{-newtype Percentile = Percentile { unPercentile :: Int }-}
87− {-deriving (Eq, Show, Read, Generic)-}
src/Servant/QuickCheck/Internal/Equality.hsnew file+58
@@ -0,0 +1,58 @@
1+module Servant.QuickCheck.Internal.Equality where
2+
3+import Data.Aeson (Value, decode, decodeStrict)
4+import Data.ByteString (ByteString)
5+import qualified Data.ByteString.Lazy as LB
6+import Data.Function (on)
7+import Network.HTTP.Client (Response, responseBody)
8+import Data.Semigroup (Semigroup (..))
9+import Prelude.Compat
10+
11+newtype ResponseEquality b
12+ = ResponseEquality { getResponseEquality :: Response b -> Response b -> Bool }
13+
14+instance Semigroup (ResponseEquality b) where
15+ ResponseEquality a <> ResponseEquality b = ResponseEquality $ \x y ->
16+ a x y && b x y
17+
18+instance Monoid (ResponseEquality b) where
19+ mempty = ResponseEquality $ \_ _ -> True
20+ mappend = (<>)
21+
22+-- | Use `Eq` instance for `Response`
23+--
24+-- /Since 0.0.0.0/
25+allEquality :: Eq b => ResponseEquality b
26+allEquality = ResponseEquality (==)
27+
28+-- | ByteString `Eq` instance over the response body.
29+--
30+-- /Since 0.0.0.0/
31+bodyEquality :: Eq b => ResponseEquality b
32+bodyEquality = ResponseEquality ((==) `on` responseBody)
33+
34+-- | Equality as 'Value'. This means that if two bodies are equal as JSON
35+-- (e.g., insignificant whitespace difference) they are considered equal.
36+--
37+-- /Since 0.0.3.0/
38+jsonEquality :: (JsonEq b) => ResponseEquality b
39+jsonEquality = ResponseEquality (jsonEq `on` responseBody)
40+
41+class JsonEq a where
42+ decode' :: a -> Maybe Value
43+ jsonEq :: a -> a -> Bool
44+ jsonEq first second = compareDecodedResponses (decode' first) (decode' second)
45+
46+instance JsonEq LB.ByteString where
47+ decode' = decode
48+
49+instance JsonEq ByteString where
50+ decode' = decodeStrict
51+
52+compareDecodedResponses :: Maybe Value -> Maybe Value -> Bool
53+compareDecodedResponses resp1 resp2 =
54+ case resp1 of
55+ Nothing -> False -- if decoding fails we assume failure
56+ (Just r1) -> case resp2 of
57+ Nothing -> False -- another decode failure
58+ (Just r2) -> r1 == r2
src/Servant/QuickCheck/Internal/ErrorTypes.hsnew file+82
@@ -0,0 +1,82 @@
1+{-# LANGUAGE CPP #-}
2+module Servant.QuickCheck.Internal.ErrorTypes where
3+
4+import Control.Exception (Exception (..))
5+import qualified Data.ByteString.Lazy as LBS
6+import Data.String.Conversions (cs)
7+import qualified Data.Text as T
8+import Data.Typeable (Typeable)
9+import GHC.Generics (Generic)
10+import qualified Network.HTTP.Client as C
11+import Network.HTTP.Types (Header, statusCode)
12+import Text.PrettyPrint
13+
14+import Prelude.Compat hiding ((<>))
15+
16+data PredicateFailure
17+ = PredicateFailure T.Text (Maybe C.Request) (C.Response LBS.ByteString)
18+ deriving (Typeable, Generic)
19+
20+instance Exception ServerEqualityFailure where
21+
22+instance Show PredicateFailure where
23+ show = render . prettyPredicateFailure
24+
25+
26+data ServerEqualityFailure
27+ = ServerEqualityFailure C.Request (C.Response LBS.ByteString) (C.Response LBS.ByteString)
28+ deriving (Typeable, Generic)
29+
30+instance Show ServerEqualityFailure where
31+ show = render . prettyServerEqualityFailure
32+
33+
34+instance Exception PredicateFailure where
35+
36+-- * Pretty printing
37+
38+prettyHeaders :: [Header] -> Doc
39+prettyHeaders hdrs = vcat $ prettyHdr <$> hdrs
40+ where
41+ prettyHdr (hn, h) = text (show hn) <> colon <+> text (show h)
42+
43+prettyReq :: C.Request -> Doc
44+prettyReq r =
45+ text "Request:" $$ (nest 5 $
46+ text "Method:" <+> (nest 5 $ text . show $ C.method r)
47+ $$ text "Path:" <+> (nest 5 $ text . cs $ C.path r)
48+ $$ text "Headers:" <+> (nest 5 $ prettyHeaders $ C.requestHeaders r)
49+ $$ text "Body:" <+> (nest 5 $ text . getReqBody $ C.requestBody r))
50+ where
51+ getReqBody (C.RequestBodyLBS lbs ) = cs lbs
52+ getReqBody (C.RequestBodyBS bs ) = cs bs
53+ getReqBody _ = error "expected bytestring body"
54+
55+prettyResp :: C.Response LBS.ByteString -> Doc
56+prettyResp r =
57+ text "Response:" $$ (nest 5 $
58+ text "Status code:" <+> (nest 5 $ text . show . statusCode $ C.responseStatus r)
59+ $$ text "Headers:" $$ (nest 10 $ prettyHeaders $ C.responseHeaders r)
60+ $$ text "Body:" <+> (nest 5 $ text . cs $ C.responseBody r))
61+
62+
63+
64+prettyServerEqualityFailure :: ServerEqualityFailure -> Doc
65+prettyServerEqualityFailure (ServerEqualityFailure req resp1 resp2) =
66+ text "Server equality failed" $$ (nest 5 $
67+ prettyReq req
68+ $$ prettyResp resp1
69+ $$ prettyResp resp2)
70+
71+
72+prettyPredicateFailure :: PredicateFailure -> Doc
73+prettyPredicateFailure (PredicateFailure predicate req resp) =
74+ text "Predicate failed" $$ (nest 5 $
75+ text "Predicate:" <+> (text $ T.unpack predicate)
76+ $$ r
77+ $$ prettyResp resp)
78+ where
79+ r = case req of
80+ Nothing -> text ""
81+ Just v -> prettyReq v
82+
src/Servant/QuickCheck/Internal/HasGenRequest.hsnew file+197
@@ -0,0 +1,197 @@
1+{-# LANGUAGE CPP #-}
2+{-# LANGUAGE PolyKinds #-}
3+module Servant.QuickCheck.Internal.HasGenRequest where
4+
5+import Data.String (fromString)
6+import Data.String.Conversions (cs)
7+import GHC.TypeLits (KnownSymbol, Nat, symbolVal)
8+import Network.HTTP.Client (Request, RequestBody (..), host, method, path,
9+ port, queryString, requestBody, requestHeaders,
10+ secure, defaultRequest)
11+import Network.HTTP.Media (renderHeader)
12+import Prelude.Compat
13+import Servant
14+import Servant.API.ContentTypes (AllMimeRender (..))
15+import Servant.Client (BaseUrl (..), Scheme (..))
16+import Test.QuickCheck (Arbitrary (..), Gen, elements, frequency)
17+
18+import qualified Data.ByteString as BS
19+import qualified Data.ByteString.Internal as BS (c2w)
20+
21+
22+-- -----------------------------------------------------------------------------
23+-- runGenRequest
24+
25+-- | This function returns a QuickCheck `Gen a` when passed a servant API value,
26+-- typically a `Proxy API`. The generator returned is a function
27+-- that accepts a `BaseUrl` and returns a `Request`, which can then be used
28+-- to issue network requests. This `Gen` type makes it easier to compare distinct
29+-- APIs across different `BaseUrl`s.
30+runGenRequest :: HasGenRequest a => Proxy a -> Gen (BaseUrl -> Request)
31+runGenRequest = snd . genRequest
32+
33+
34+-- -----------------------------------------------------------------------------
35+-- HasGenRequest
36+
37+-- | This is the core Servant-Quickcheck generator, which, when given a `Proxy API`
38+-- will return a pair of `Int` and `Gen a`, where `a` is a function from
39+-- `BaseUrl` to a `Network.Http.Client.Request`. The `Int` is a weight for the
40+-- QuickCheck `frequency` function which ensures a random distribution across
41+-- all endpoints in an API.
42+class HasGenRequest a where
43+ genRequest :: Proxy a -> (Int, Gen (BaseUrl -> Request))
44+
45+
46+instance (HasGenRequest a, HasGenRequest b) => HasGenRequest (a :<|> b) where
47+ genRequest _
48+ = (lf + rf, frequency [l, r])
49+ where
50+ l@(lf, _) = genRequest (Proxy :: Proxy a)
51+ r@(rf, _) = genRequest (Proxy :: Proxy b)
52+
53+
54+instance (KnownSymbol path, HasGenRequest b) => HasGenRequest (path :> b) where
55+ genRequest _ = (oldf, do
56+ old' <- old
57+ return $ \burl -> let r = old' burl
58+ oldPath = path r
59+ oldPath' = BS.dropWhile (== BS.c2w '/') oldPath
60+ paths = filter (not . BS.null) [new, oldPath']
61+ in r { path = "/" <> BS.intercalate "/" paths })
62+ where
63+ (oldf, old) = genRequest (Proxy :: Proxy b)
64+ new = cs $ symbolVal (Proxy :: Proxy path)
65+
66+instance HasGenRequest EmptyAPI where
67+ genRequest _ = (0, error "EmptyAPIs cannot be queried.")
68+
69+instance HasGenRequest api => HasGenRequest (Summary d :> api) where
70+ genRequest _ = genRequest (Proxy :: Proxy api)
71+
72+instance HasGenRequest api => HasGenRequest (Description d :> api) where
73+ genRequest _ = genRequest (Proxy :: Proxy api)
74+
75+instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
76+ => HasGenRequest (Capture' mods x c :> b) where
77+ genRequest _ = (oldf, do
78+ old' <- old
79+ new' <- toUrlPiece <$> new
80+ return $ \burl -> let r = old' burl in r { path = cs new' <> path r })
81+ where
82+ (oldf, old) = genRequest (Proxy :: Proxy b)
83+ new = arbitrary :: Gen c
84+
85+instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
86+ => HasGenRequest (CaptureAll x c :> b) where
87+ genRequest _ = (oldf, do
88+ old' <- old
89+ new' <- fmap (cs . toUrlPiece) <$> new
90+ let new'' = BS.intercalate "/" new'
91+ return $ \burl -> let r = old' burl in r { path = new'' <> path r })
92+ where
93+ (oldf, old) = genRequest (Proxy :: Proxy b)
94+ new = arbitrary :: Gen [c]
95+
96+instance (Arbitrary c, KnownSymbol h, HasGenRequest b, ToHttpApiData c)
97+ => HasGenRequest (Header' mods h c :> b) where
98+ genRequest _ = (oldf, do
99+ old' <- old
100+ new' <- toUrlPiece <$> new -- TODO: generate lenient or/and optional
101+ return $ \burl -> let r = old' burl in r {
102+ requestHeaders = (hdr, cs new') : requestHeaders r })
103+ where
104+ (oldf, old) = genRequest (Proxy :: Proxy b)
105+ hdr = fromString $ symbolVal (Proxy :: Proxy h)
106+ new = arbitrary :: Gen c
107+
108+instance (AllMimeRender x c, Arbitrary c, HasGenRequest b)
109+ => HasGenRequest (ReqBody' mods x c :> b) where
110+ genRequest _ = (oldf, do
111+ old' <- old -- TODO: generate lenient
112+ new' <- new
113+ (ct, bd) <- elements $ allMimeRender (Proxy :: Proxy x) new'
114+ return $ \burl -> let r = old' burl in r {
115+ requestBody = RequestBodyLBS bd
116+ , requestHeaders = ("Content-Type", renderHeader ct) : requestHeaders r
117+ })
118+ where
119+ (oldf, old) = genRequest (Proxy :: Proxy b)
120+ new = arbitrary :: Gen c
121+
122+instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
123+ => HasGenRequest (QueryParam' mods x c :> b) where
124+ genRequest _ = (oldf, do
125+ new' <- new -- TODO: generate lenient or/and optional
126+ old' <- old
127+ return $ \burl -> let r = old' burl
128+ newExpr = param <> "=" <> cs (toQueryParam new')
129+ qs = queryString r in r {
130+ queryString = if BS.null qs then newExpr else newExpr <> "&" <> qs })
131+ where
132+ (oldf, old) = genRequest (Proxy :: Proxy b)
133+ param = cs $ symbolVal (Proxy :: Proxy x)
134+ new = arbitrary :: Gen c
135+
136+instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
137+ => HasGenRequest (QueryParams x c :> b) where
138+ genRequest _ = (oldf, do
139+ new' <- new
140+ old' <- old
141+ return $ \burl -> let r = old' burl in r {
142+ queryString = queryString r
143+ <> if length new' > 0 then fold (toParam <$> new') else ""})
144+ where
145+ (oldf, old) = genRequest (Proxy :: Proxy b)
146+ param = cs $ symbolVal (Proxy :: Proxy x)
147+ new = arbitrary :: Gen [c]
148+ toParam c = param <> "[]=" <> cs (toQueryParam c)
149+ fold = foldr1 (\a b -> a <> "&" <> b)
150+
151+instance (KnownSymbol x, HasGenRequest b)
152+ => HasGenRequest (QueryFlag x :> b) where
153+ genRequest _ = (oldf, do
154+ old' <- old
155+ return $ \burl -> let r = old' burl
156+ qs = queryString r in r {
157+ queryString = if BS.null qs then param else param <> "&" <> qs })
158+ where
159+ (oldf, old) = genRequest (Proxy :: Proxy b)
160+ param = cs $ symbolVal (Proxy :: Proxy x)
161+
162+instance (ReflectMethod method)
163+ => HasGenRequest (Verb (method :: k) (status :: Nat) (cts :: [*]) a) where
164+ genRequest _ = (1, return $ \burl -> defaultRequest
165+ { host = cs $ baseUrlHost burl
166+ , port = baseUrlPort burl
167+ , secure = baseUrlScheme burl == Https
168+ , method = reflectMethod (Proxy :: Proxy method)
169+ })
170+
171+instance (ReflectMethod method)
172+ => HasGenRequest (NoContentVerb (method :: k)) where
173+ genRequest _ = (1, return $ \burl -> defaultRequest
174+ { host = cs $ baseUrlHost burl
175+ , port = baseUrlPort burl
176+ , secure = baseUrlScheme burl == Https
177+ , method = reflectMethod (Proxy :: Proxy method)
178+ })
179+
180+instance (HasGenRequest a) => HasGenRequest (RemoteHost :> a) where
181+ genRequest _ = genRequest (Proxy :: Proxy a)
182+
183+instance (HasGenRequest a) => HasGenRequest (IsSecure :> a) where
184+ genRequest _ = genRequest (Proxy :: Proxy a)
185+
186+instance (HasGenRequest a) => HasGenRequest (HttpVersion :> a) where
187+ genRequest _ = genRequest (Proxy :: Proxy a)
188+
189+instance (HasGenRequest a) => HasGenRequest (Vault :> a) where
190+ genRequest _ = genRequest (Proxy :: Proxy a)
191+
192+instance (HasGenRequest a) => HasGenRequest (WithNamedContext x y a) where
193+ genRequest _ = genRequest (Proxy :: Proxy a)
194+
195+-- TODO: Try logging in
196+instance (HasGenRequest a) => HasGenRequest (BasicAuth x y :> a) where
197+ genRequest _ = genRequest (Proxy :: Proxy a)
src/Servant/QuickCheck/Internal/Predicates.hs+453−124
@@ -1,132 +1,461 @@
1−-- | This module contains all logic related to constructing or using
2−-- @Predicates@.
3−{-# LANGUAGE UndecidableInstances #-}
4−{-# LANGUAGE ScopedTypeVariables #-}
51module Servant.QuickCheck.Internal.Predicates where
62
7−import Data.Aeson (ToJSON (toJSON), Value (..))
8−import Data.Proxy (Proxy (..))
9−import Data.Void
10−import Network.HTTP.Types (statusCode)
11−import Servant.Common.Req (ServantError (..))
12−import Test.QuickCheck
13−
14−
15−-- | An HList containing predicates (functions of type @a -> Bool@). This
16−-- datatype is used to represent both filters (what values to discard when
17−-- generating arguments to test an API) and tests results (what to consider a
18−-- failing response).
19−--
20−-- For both filters and test results, only the *first* predicate of the
21−-- appropriate type is used.
22−--
23−-- Use 'emptyPredicates', 'addPredicate', 'addLeftPredicate' and
24−-- 'addRightPredicate' to construct a @Predicates@.
25−data Predicates a where
26− HNil :: Predicates '[]
27− HCons :: (a -> Bool) -> Predicates b -> Predicates (a ': b)
28− HConsC :: Constraint a -> Predicates b -> Predicates (Constraint a ': b)
29−
30−class HasPredicate a b where
31− getPredicate :: Predicates a -> b -> Bool
32−
33−instance {-# OVERLAPPING #-} HasPredicate '[] a where
34− getPredicate _ = const True
35−
36−-- TODO: Find some better way of distinguishing how the predicate is being used
37−instance {-# OVERLAPPING #-} HasPredicate '[] (Either ServantError a) where
38− getPredicate _ = discard
39−
40−instance {-# OVERLAPPING #-} HasPredicate (a ': xs) a where
41− getPredicate (HCons a _) = a
42− getPredicate (HConsC _ _) = error "not impossible, but non-sensical"
43−
44−data Constraint ctx = Constraint
45− { getConstraint :: forall a . (ctx a) => a -> Bool }
46−
47−-- This is a little bit of a hack. Ideally instances would match when the
48−-- predicate is polymorphic, but that doesn't work since the polymorphic type
49−-- may have to unify with multiple distict values.
50−--
51−-- It may however be possible to define a MPTC from monomorphic to polymorphic
52−-- datatypes to avoid this issue.
53−instance {-# OVERLAPPING #-}
54− HasPredicate (Either ServantError Void ': xs) (Either ServantError a) where
55− getPredicate (HCons f _) x = case x of
56− Left e -> f (Left e)
57− Right _ -> True
58−
59−instance {-# OVERLAPPING #-} (ctx a)
60− => HasPredicate (Constraint ctx ': xs) (Either ServantError a) where
61− getPredicate (HConsC f _) x = case x of
62− Left _ -> discard -- Not clear whether checking for FailureResponse is better
63− Right v -> getConstraint f v
64− getPredicate (HCons _ _) _ = error "not impossible, but non-sensical"
65−
66−instance {-# OVERLAPPABLE #-} (ls ~ (b ': xs), HasPredicate xs a)
67− => HasPredicate ls a where
68− getPredicate (HCons _ xs) = getPredicate xs
69− getPredicate _ = error "impossible"
70−
71−-- | Add a predicate to a list of predicates. Note that the predicate may not
72−-- be polymorphic.
73−addPredicate :: (a -> Bool) -> Predicates b -> Predicates (a ': b)
74−addPredicate = HCons
75−
76−-- | Add a predicate with a class constraint.
77−--
78−-- Note that every possible argument must be an instance of that class for this
79−-- to typecheck. In other words, if the @Predicates@ is being used for return
80−-- types, every return type in the API must be an instance of the class. If
81−-- it's being used for filtering, every capture, header, body, etc. type must
82−-- be an instance of that class.
83−--
84−-- This can be used to for example test that returned JSON has certain
85−-- properties, or (via generics) that if any datatype contains a (possibly
86−-- nested) field of a particular type, it always meets certain properties.
87−addPolyPredicate :: proxy ctx -> (forall a. ctx a => a -> Bool) -> Predicates b
88− -> Predicates (Constraint ctx ': b)
89−addPolyPredicate _ p = HConsC (Constraint p)
90−
91−-- | Given a predicate over an @p :: a -> Bool@, add a predicate to the @Predicates@
92−-- list that succeeds on an @val :: Either ServantError a@ if @val@ is a
93−-- @Left@, or a @Right v@ such that @p a == True@.
94−addRightPredicate :: (a -> Bool) -> Predicates b -> Predicates (Either ServantError a ': b)
95−addRightPredicate p = addPredicate $ either (const True) p
96−
97−-- | The @Left@ analog of 'addRightPredicate'.
98−addLeftPredicate :: (ServantError -> Bool) -> Predicates b
99− -> Predicates (Either ServantError Void ': b)
100−addLeftPredicate p = addPredicate $ either p (error "impossible")
101−
102−-- | An empty list of predicates. This doesn't discard any values when used as
103−-- a filter, and doesn't fail any value when used as a condition to satisfy.
104−emptyPredicates :: Predicates '[]
105−emptyPredicates = HNil
106−
107−-- * Useful predicates
108−
109−-- | A @Predicates@ list that fails a test if the response is an HTTP 500 error.
110−never500s :: Predicates '[Either ServantError Void]
111−never500s = addLeftPredicate go emptyPredicates
112− where
113− go (FailureResponse x _ _) = statusCode x /= 500
114− go _ = True
3+import Control.Exception (catch, throw)
4+import Control.Monad (liftM2, unless, when)
5+import Data.Aeson (Object, decode)
6+import Data.Bifunctor (first)
7+import qualified Data.ByteString as SBS
8+import qualified Data.ByteString.Char8 as SBSC
9+import qualified Data.ByteString.Lazy as LBS
10+import Data.CaseInsensitive (foldCase, foldedCase, mk)
11+import Data.Either (isRight)
12+import Data.List.Split (wordsBy)
13+import Data.Maybe (fromMaybe, isJust)
14+import Data.Semigroup (Semigroup (..))
15+import qualified Data.Text as T
16+import Data.Time (UTCTime, defaultTimeLocale, parseTimeM,
17+ rfc822DateFormat)
18+import GHC.Generics (Generic)
19+import Network.HTTP.Client (Manager, Request, Response, httpLbs,
20+ method, parseRequest, requestHeaders,
21+ responseBody, responseHeaders,
22+ responseStatus)
23+import Network.HTTP.Media (matchAccept)
24+import Network.HTTP.Types (methodGet, methodHead, parseMethod,
25+ renderStdMethod, status100, status200,
26+ status201, status300, status401,
27+ status405, status500)
28+import Prelude.Compat
29+import System.Clock (Clock (Monotonic), diffTimeSpec,
30+ getTime, toNanoSecs)
31+
32+import Servant.QuickCheck.Internal.ErrorTypes
33+
34+
35+-- | [__Best Practice__]
36+--
37+-- @500 Internal Server Error@ should be avoided - it may represent some
38+-- issue with the application code, and it moreover gives the client little
39+-- indication of how to proceed or what went wrong.
40+--
41+-- This function checks that the response code is not 500.
42+--
43+-- /Since 0.0.0.0/
44+not500 :: ResponsePredicate
45+not500 = ResponsePredicate $ \resp ->
46+ when (responseStatus resp == status500) $ throw $ PredicateFailure "not500" Nothing resp
11547
116−-- | A @Predicates@ list that fails a test if the response is anything but a
117−-- top-level object (e.g., if it is an array or literal).
48+-- | [__Optional__]
11849--
119−-- Returning anything other than object is considered bad practice, as
50+-- This function checks that the response from the server does not take longer
51+-- than the specified number of nanoseconds.
52+--
53+-- /Since 0.0.2.1/
54+notLongerThan :: Integer -> RequestPredicate
55+notLongerThan maxAllowed
56+ = RequestPredicate $ \req mgr -> do
57+ start <- getTime Monotonic
58+ resp <- httpLbs req mgr
59+ end <- getTime Monotonic
60+ when (toNanoSecs (end `diffTimeSpec` start) > maxAllowed) $
61+ throw $ PredicateFailure "notLongerThan" (Just req) resp
62+ return []
63+
64+-- | [__Best Practice__]
65+--
66+-- Returning anything other than an object when returning JSON is considered
67+-- bad practice, as:
12068--
12169-- (1) it is hard to modify the returned value while maintaining backwards
122−-- compatibility;
123−-- (2) many older tools do not support top-level arrays;
70+-- compatibility
71+-- (2) many older tools do not support top-level arrays
12472-- (3) whether top-level numbers, booleans, or strings are valid JSON depends
125−-- on what RFC you're going by;
126−-- (4) there are security issues with top-level arrays.
127−onlyJsonObjects :: Predicates '[Constraint ToJSON]
128−onlyJsonObjects = addPolyPredicate (Proxy :: Proxy ToJSON) go emptyPredicates
73+-- on what RFC you're going by
74+-- (4) there are security issues with top-level arrays
75+--
76+-- This function checks that any @application/json@ responses only return JSON
77+-- objects (and not arrays, strings, numbers, or booleans) at the top level.
78+--
79+-- __References__:
80+--
81+-- * JSON Grammar: <https://tools.ietf.org/html/rfc7159#section-2 RFC 7159 Section 2>
82+-- * JSON Grammar: <https://tools.ietf.org/html/rfc4627#section-2 RFC 4627 Section 2>
83+--
84+-- /Since 0.0.0.0/
85+onlyJsonObjects :: ResponsePredicate
86+onlyJsonObjects
87+ = ResponsePredicate (\resp -> do
88+ case lookup "content-type" (first foldedCase <$> responseHeaders resp) of
89+ Nothing -> return ()
90+ Just ctype -> when ("application/json" `SBS.isPrefixOf` ctype) $ do
91+ case (decode (responseBody resp) :: Maybe Object) of
92+ Nothing -> throw $ PredicateFailure "onlyJsonObjects" Nothing resp
93+ Just _ -> return ())
94+
95+-- | __Optional__
96+--
97+-- When creating a new resource, it is good practice to provide a @Location@
98+-- header with a link to the created resource.
99+--
100+-- This function checks that every @201 Created@ response contains a @Location@
101+-- header, and that the link in it responds with a 2XX response code to @GET@
102+-- requests.
103+--
104+-- This is considered optional because other means of linking to the resource
105+-- (e.g. via the response body) are also acceptable; linking to the resource in
106+-- some way is considered best practice.
107+--
108+-- __References__:
109+--
110+-- * 201 Created: <https://tools.ietf.org/html/rfc7231#section-6.3.2 RFC 7231 Section 6.3.2>
111+-- * Location header: <https://tools.ietf.org/html/rfc7231#section-7.1.2 RFC 7231 Section 7.1.2>
112+--
113+-- /Since 0.0.0.0/
114+createContainsValidLocation :: RequestPredicate
115+createContainsValidLocation
116+ = RequestPredicate $ \req mgr -> do
117+ let n = "createContainsValidLocation"
118+ resp <- httpLbs req mgr
119+ if responseStatus resp == status201
120+ then case lookup "Location" $ responseHeaders resp of
121+ Nothing -> throw $ PredicateFailure n (Just req) resp
122+ Just l -> case parseRequest $ SBSC.unpack l of
123+ Nothing -> throw $ PredicateFailure n (Just req) resp
124+ Just x -> do
125+ resp2 <- httpLbs x mgr
126+ status2XX (Just req) resp2 n
127+ return [resp, resp2]
128+ else return [resp]
129+
130+-- | [__Optional__]
131+--
132+-- The @Last-Modified@ header represents the time a resource was last
133+-- modified. It is used to drive caching and conditional requests.
134+--
135+-- When using this mechanism, the server adds the @Last-Modified@ header to
136+-- responses. Clients may then make requests with the @If-Modified-Since@
137+-- header to conditionally request resources. If the resource has not
138+-- changed since that date, the server responds with a status code of 304
139+-- (@Not Modified@) without a response body.
140+--
141+-- The @Last-Modified@ header can also be used in conjunction with the
142+-- @If-Unmodified-Since@ header to drive optimistic concurrency.
143+--
144+-- The @Last-Modified@ date must be in RFC 822 format.
145+--
146+-- __References__:
147+--
148+-- * 304 Not Modified: <https://tools.ietf.org/html/rfc7232#section-4.1 RFC 7232 Section 4.1>
149+-- * Last-Modified header: <https://tools.ietf.org/html/rfc7232#section-2.2 RFC 7232 Section 2.2>
150+-- * If-Modified-Since header: <https://tools.ietf.org/html/rfc7232#section-3.3 RFC 7232 Section 3.3>
151+-- * If-Unmodified-Since header: <https://tools.ietf.org/html/rfc7232#section-3.4 RFC 7232 Section 3.4>
152+-- * Date format: <https://tools.ietf.org/html/rfc2616#section-3.3 RFC 2616 Section 3.3>
153+--
154+-- /Since 0.0.2.1/
155+getsHaveLastModifiedHeader :: RequestPredicate
156+getsHaveLastModifiedHeader
157+ = RequestPredicate $ \req mgr ->
158+ if (method req == methodGet)
159+ then do
160+ resp <- httpLbs req mgr
161+ unless (hasValidHeader "Last-Modified" isRFC822Date resp) $ do
162+ throw $ PredicateFailure "getsHaveLastModifiedHeader" (Just req) resp
163+ return [resp]
164+ else return []
165+
166+
167+
168+-- | [__RFC Compliance__]
169+--
170+-- When an HTTP request has a method that is not allowed,
171+-- a 405 response should be returned. Additionally, it is good practice to
172+-- return an @Allow@
173+-- header with the list of allowed methods.
174+--
175+-- This function checks that every @405 Method Not Allowed@ response contains
176+-- an @Allow@ header with a list of standard HTTP methods.
177+--
178+-- Note that 'servant' itself does not currently set the @Allow@ headers.
179+--
180+-- __References__:
181+--
182+-- * @Allow@ header: <https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html RFC 2616 Section 14.7>
183+-- * Status 405: <https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html RFC 2616 Section 10.4.6>
184+-- * Servant Allow header issue: <https://github.com/haskell-servant/servant/issues/489 Issue #489>
185+--
186+-- /Since 0.0.0.0/
187+notAllowedContainsAllowHeader :: RequestPredicate
188+notAllowedContainsAllowHeader
189+ = RequestPredicate $ \req mgr -> do
190+ let reqs = [ req { method = renderStdMethod m } | m <- [minBound .. maxBound]
191+ , renderStdMethod m /= method req ]
192+ resp <- mapM (flip httpLbs mgr) reqs
193+
194+ case filter pred' (zip reqs resp) of
195+ (x:_) -> throw $ PredicateFailure "notAllowedContainsAllowHeader" (Just $ fst x) (snd x)
196+ [] -> return resp
197+ where
198+ pred' (_, resp) = responseStatus resp == status405 && not (hasValidHeader "Allow" go resp)
199+ where
200+ go x = all (\y -> isRight $ parseMethod $ SBSC.pack y)
201+ $ wordsBy (`elem` (", " :: [Char])) (SBSC.unpack x)
202+
203+
204+-- | [__RFC Compliance__]
205+--
206+-- When a request contains an @Accept@ header, the server must either return
207+-- content in one of the requested representations, or respond with @406 Not
208+-- Acceptable@.
209+--
210+-- This function checks that every *successful* response has a @Content-Type@
211+-- header that matches the @Accept@ header. It does *not* check that the server
212+-- matches the quality descriptions of the @Accept@ header correctly.
213+--
214+-- __References__:
215+--
216+-- * @Accept@ header: <https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html RFC 2616 Section 14.1>
217+--
218+-- /Since 0.0.0.0/
219+honoursAcceptHeader :: RequestPredicate
220+honoursAcceptHeader
221+ = RequestPredicate $ \req mgr -> do
222+ resp <- httpLbs req mgr
223+ let scode = responseStatus resp
224+ sctype = lookup "Content-Type" $ responseHeaders resp
225+ sacc = fromMaybe "*/*" $ lookup "Accept" (requestHeaders req)
226+ if status100 < scode && scode < status300
227+ then if isJust $ sctype >>= \x -> matchAccept [x] sacc
228+ then throw $ PredicateFailure "honoursAcceptHeader" (Just req) resp
229+ else return [resp]
230+ else return [resp]
231+
232+
233+-- | [__Best Practice__]
234+--
235+-- Whether or not a representation should be cached, it is good practice to
236+-- have a @Cache-Control@ header for @GET@ requests. If the representation
237+-- should not be cached, used @Cache-Control: no-cache@.
238+--
239+-- This function checks that @GET@ responses have @Cache-Control@ header.
240+-- It does NOT currently check that the header is valid.
241+--
242+-- __References__:
243+--
244+-- * @Cache-Control@ header: <https://tools.ietf.org/html/rfc7234#section-5.2 RFC 7234 Section 5.2>
245+--
246+-- /Since 0.0.0.0/
247+getsHaveCacheControlHeader :: RequestPredicate
248+getsHaveCacheControlHeader
249+ = RequestPredicate $ \req mgr ->
250+ if (method req == methodGet)
251+ then do
252+ resp <- httpLbs req mgr
253+ unless (hasValidHeader "Cache-Control" (const True) resp) $ do
254+ throw $ PredicateFailure "getsHaveCacheControlHeader" (Just req) resp
255+ return [resp]
256+ else return []
257+
258+-- | [__Best Practice__]
259+--
260+-- Like 'getsHaveCacheControlHeader', but for @HEAD@ requests.
261+--
262+-- /Since 0.0.0.0/
263+headsHaveCacheControlHeader :: RequestPredicate
264+headsHaveCacheControlHeader
265+ = RequestPredicate $ \req mgr ->
266+ if (method req == methodHead)
267+ then do
268+ resp <- httpLbs req mgr
269+ unless (hasValidHeader "Cache-Control" (const True) resp) $
270+ throw $ PredicateFailure "headsHaveCacheControlHeader" (Just req) resp
271+ return [resp]
272+ else return []
273+{-
274+-- |
275+--
276+-- If the original request modifies the resource, this function makes two
277+-- requests:
278+--
279+-- (1) Once, with the original request and a future date as the
280+-- @If-Unmodified-Since@, which is expected to succeed.
281+-- (2) Then with the original request again, with a @If-Unmodified-Since@
282+-- safely in the past. Since presumably the representation has been changed
283+-- recently (by the first request), this is expected to fail with @412
284+-- Precondition Failure@.
285+--
286+-- Note that the heuristic used to guess whether the original request modifies
287+-- a resource is simply whether the method is @PUT@ or @PATCH@, which may be
288+-- incorrect in certain circumstances.
289+supportsIfUnmodifiedSince :: Predicate b Bool
290+supportsIfUnmodifiedSince
291+ = ResponsePredicate "supportsIfUnmodifiedSince" _
292+
293+-- | @OPTIONS@ responses should contain an @Allow@ header with the list of
294+-- allowed methods.
295+--
296+-- If a request is an @OPTIONS@ request, and if the response is a successful
297+-- one, this function checks the response for an @Allow@ header. It fails if:
298+--
299+-- (1) There is no @Allow@ header
300+-- (2) The @Allow@ header does not have standard HTTP methods in the correct
301+-- format
302+-- (3) Making a request to the same URL with one of those methods results in
303+-- a 404 or 405.
304+optionsContainsValidAllow :: Predicate b Bool
305+optionsContainsValidAllow
306+ = ResponsePredicate "optionsContainsValidAllow" _
307+
308+-- | Link headers are a standardized way of presenting links that may be
309+-- relevant to a client.
310+--
311+-- This function checks that any @Link@ headers have values in the correct
312+-- format.
313+--
314+-- __References__:
315+--
316+-- * <https://tools.ietf.org/html/rfc5988 RFC 5988 Section 5>
317+linkHeadersAreValid :: Predicate b Bool
318+linkHeadersAreValid
319+ = ResponsePredicate "linkHeadersAreValid" _
320+
321+-}
322+-- | [__RFC Compliance__]
323+--
324+-- Any @401 Unauthorized@ response must include a @WWW-Authenticate@ header.
325+--
326+-- This function checks that, if a response has status code 401, it contains a
327+-- @WWW-Authenticate@ header.
328+--
329+-- __References__:
330+--
331+-- * @WWW-Authenticate@ header: <https://tools.ietf.org/html/rfc7235#section-4.1 RFC 7235 Section 4.1>
332+--
333+-- /Since 0.0.0.0/
334+unauthorizedContainsWWWAuthenticate :: ResponsePredicate
335+unauthorizedContainsWWWAuthenticate
336+ = ResponsePredicate $ \resp ->
337+ if responseStatus resp == status401
338+ then unless (hasValidHeader "WWW-Authenticate" (const True) resp) $
339+ throw $ PredicateFailure "unauthorizedContainsWWWAuthenticate" Nothing resp
340+ else return ()
341+
342+
343+-- | [__RFC Compliance__]
344+--
345+-- [An HTML] document will start with exactly this string: <!DOCTYPE html>
346+--
347+-- This function checks that HTML documents (those with `Content-Type: text/html...`)
348+-- include a DOCTYPE declaration at the top. We do not enforce capital case for the string `DOCTYPE`.
349+--
350+-- __References__:
351+--
352+-- * HTML5 Doctype: <https://tools.ietf.org/html/rfc7992#section-6.1 RFC 7992 Section 6.1>
353+-- /Since 0.3.0.0/
354+htmlIncludesDoctype :: ResponsePredicate
355+htmlIncludesDoctype
356+ = ResponsePredicate $ \resp ->
357+ if hasValidHeader "Content-Type" (SBS.isPrefixOf . foldCase $ "text/html") resp
358+ then do
359+ let htmlContent = foldCase . LBS.take 20 $ responseBody resp
360+ unless (LBS.isPrefixOf (foldCase "<!doctype html>") htmlContent) $
361+ throw $ PredicateFailure "htmlIncludesDoctype" Nothing resp
362+ else return ()
363+
364+-- * Predicate logic
365+
366+-- The idea with all this footwork is to not waste any requests. Rather than
367+-- generating new requests and only applying one predicate to the response, we
368+-- apply as many predicates as possible.
369+--
370+-- Still, this is all kind of ugly.
371+
372+-- | A predicate that depends only on the response.
373+--
374+-- /Since 0.0.0.0/
375+newtype ResponsePredicate = ResponsePredicate
376+ { getResponsePredicate :: Response LBS.ByteString -> IO ()
377+ } deriving (Generic)
378+
379+instance Semigroup ResponsePredicate where
380+ ResponsePredicate a <> ResponsePredicate b = ResponsePredicate $ \x -> a x >> b x
381+
382+instance Monoid ResponsePredicate where
383+ mempty = ResponsePredicate $ const $ return ()
384+ mappend = (<>)
385+
386+-- | A predicate that depends on both the request and the response.
387+--
388+-- /Since 0.0.0.0/
389+newtype RequestPredicate = RequestPredicate
390+ { getRequestPredicate :: Request -> Manager -> IO [Response LBS.ByteString]
391+ } deriving (Generic)
392+
393+-- TODO: This isn't actually a monoid
394+instance Monoid RequestPredicate where
395+ mempty = RequestPredicate (\r m -> httpLbs r m >>= \x -> return ([x]))
396+ mappend = (<>)
397+
398+-- TODO: This isn't actually a monoid
399+instance Semigroup RequestPredicate where
400+ RequestPredicate a <> RequestPredicate b = RequestPredicate $ \r mgr ->
401+ liftM2 (<>) (a r mgr) (b r mgr)
402+
403+-- | A set of predicates. Construct one with 'mempty' and '<%>'.
404+data Predicates = Predicates
405+ { requestPredicates :: RequestPredicate
406+ , responsePredicates :: ResponsePredicate
407+ } deriving (Generic)
408+
409+instance Semigroup Predicates where
410+ a <> b = Predicates (requestPredicates a <> requestPredicates b)
411+ (responsePredicates a <> responsePredicates b)
412+
413+instance Monoid Predicates where
414+ mempty = Predicates mempty mempty
415+ mappend = (<>)
416+
417+class JoinPreds a where
418+ joinPreds :: a -> Predicates -> Predicates
419+
420+instance JoinPreds (RequestPredicate ) where
421+ joinPreds p (Predicates x y) = Predicates (p <> x) y
422+
423+instance JoinPreds (ResponsePredicate ) where
424+ joinPreds p (Predicates x y) = Predicates x (p <> y)
425+
426+-- | Adds a new predicate (either `ResponsePredicate` or `RequestPredicate`) to
427+-- the existing predicates.
428+--
429+-- > not500 <%> onlyJsonObjects <%> empty
430+--
431+-- /Since 0.0.0.0/
432+(<%>) :: JoinPreds a => a -> Predicates -> Predicates
433+(<%>) = joinPreds
434+infixr 6 <%>
435+
436+finishPredicates :: Predicates -> Request -> Manager -> IO (Maybe PredicateFailure)
437+finishPredicates p req mgr = go `catch` \(e :: PredicateFailure) -> return $ Just e
129438 where
130− go x = case toJSON x of
131− Object _ -> True
132− _ -> False
439+ go = do
440+ resps <- getRequestPredicate (requestPredicates p) req mgr
441+ mapM_ (getResponsePredicate $ responsePredicates p) resps
442+ return Nothing
443+
444+-- * helpers
445+
446+hasValidHeader :: SBS.ByteString -> (SBS.ByteString -> Bool) -> Response b -> Bool
447+hasValidHeader hdr p r = case lookup (mk hdr) (responseHeaders r) of
448+ Nothing -> False
449+ Just v -> p v
450+
451+isRFC822Date :: SBS.ByteString -> Bool
452+isRFC822Date s
453+ = case parseTimeM True defaultTimeLocale rfc822DateFormat (SBSC.unpack s) of
454+ Nothing -> False
455+ Just (_ :: UTCTime) -> True
456+
457+status2XX :: Monad m => Maybe Request -> Response LBS.ByteString -> T.Text -> m ()
458+status2XX mreq resp t
459+ | status200 <= responseStatus resp && responseStatus resp < status300
460+ = return ()
461+ | otherwise = throw $ PredicateFailure t mreq resp
src/Servant/QuickCheck/Internal/QuickCheck.hs+161−127
@@ -1,57 +1,54 @@
1−-- | This module contains wrappers around lower-level functionality.
1+{-# LANGUAGE RecordWildCards #-}
2+{-# LANGUAGE CPP #-}
23module Servant.QuickCheck.Internal.QuickCheck where
34
4−import Control.Concurrent (threadDelay)
5−import Control.Concurrent.MVar (modifyMVar_, readMVar)
6−import Control.Monad (replicateM_)
5+import Control.Concurrent (tryReadMVar, newEmptyMVar, tryPutMVar)
6+import Control.Monad (unless)
7+import qualified Data.ByteString.Lazy as LBS
78import Data.Proxy (Proxy)
8−import Data.Void (Void)
9−import Network.HTTP.Client (Manager, defaultManagerSettings,
10− newManager)
11−import Network.HTTP.Client (managerModifyRequest, getUri)
9+import qualified Network.HTTP.Client as C
1210import Network.Wai.Handler.Warp (withApplication)
13−import Servant (HasServer, Server, serve)
14−import Servant.Client (BaseUrl (..), Client, HasClient,
15− Scheme (..), ServantError, client)
16−import System.IO (hPutStrLn, hFlush)
17−import System.IO.Temp (withSystemTempFile)
18−import System.Mem (performGC)
19−import System.Process (callCommand)
11+import Prelude.Compat
12+import Servant (Context (EmptyContext), HasServer,
13+ Server, serveWithContext)
14+#if MIN_VERSION_servant_server(0,18,0)
15+import Servant (DefaultErrorFormatters, ErrorFormatters, HasContextEntry, type (.++))
16+#endif
17+import Servant.Client (BaseUrl (..), Scheme (..))
18+import System.IO.Unsafe (unsafePerformIO)
2019import Test.Hspec (Expectation, expectationFailure)
21−import Test.QuickCheck (Args (..), Property, Result (..),
22− Testable, property,
23− quickCheckWithResult, stdArgs)
20+import Test.QuickCheck (Args (..), Result (..), quickCheckWithResult)
21+import Test.QuickCheck.Monadic (assert, forAllM, monadicIO, monitor,
22+ run)
23+import Test.QuickCheck.Property (counterexample)
2424
25−import Servant.QuickCheck.Internal.Testable
25+import Servant.QuickCheck.Internal.Equality
26+import Servant.QuickCheck.Internal.ErrorTypes
27+import Servant.QuickCheck.Internal.HasGenRequest
2628import Servant.QuickCheck.Internal.Predicates
27−import Servant.QuickCheck.Internal.Benchmarking
2829
2930
3031-- | Start a servant application on an open port, run the provided function,
3132-- then stop the application.
33+--
34+-- /Since 0.0.0.0/
3235withServantServer :: HasServer a '[] => Proxy a -> IO (Server a)
3336 -> (BaseUrl -> IO r) -> IO r
34−withServantServer api server t
35− = withApplication (return . serve api =<< server) $ \port ->
36− t (BaseUrl Http "localhost" port "")
37+withServantServer api = withServantServerAndContext api EmptyContext
3738
38−-- | A QuickCheck 'Property' that randomly generates arguments (captures, query
39−-- params, request bodies, headers, etc.) expected by endpoints of a server,
40−-- and makes requests to the servers running in the two provided URLs in the
41−-- same order, failing if they do not return the same response.
39+-- | Like 'withServantServer', but allows passing in a 'Context' to the
40+-- application.
4241--
43−-- Evidently, if the behaviour of the server is expected to be
44−-- non-deterministic, this function may produce spurious failures.
45−--
46−-- Note that this QuickCheck 'Property' does IO; interleaving it with other IO
47−-- actions will not work. It is provided so that it can be used with QuickCheck
48−-- functions such as 'quickCheckWith'. For most use cases, you should use
49−-- @serversEqual@ or @servantServersEqual@.
50−serversEqualProperty :: (HasClient a, Testable (ShouldMatch (Client a)))
51− => Proxy a -> Manager -> BaseUrl -> BaseUrl -> Property
52−serversEqualProperty api mgr burl1 burl2 = property $ ShouldMatch c1 c2
53− where c1 = client api burl1 mgr
54− c2 = client api burl2 mgr
42+-- /Since 0.0.0.0/
43+#if MIN_VERSION_servant_server(0,18,0)
44+withServantServerAndContext :: (HasServer a ctx, HasContextEntry (ctx .++ DefaultErrorFormatters) ErrorFormatters)
45+#else
46+withServantServerAndContext :: HasServer a ctx
47+#endif
48+ => Proxy a -> Context ctx -> IO (Server a) -> (BaseUrl -> IO r) -> IO r
49+withServantServerAndContext api ctx server t
50+ = withApplication (return . serveWithContext api ctx =<< server) $ \port ->
51+ t (BaseUrl Http "localhost" port "")
5552
5653-- | Check that the two servers running under the provided @BaseUrl@s behave
5754-- identically by randomly generating arguments (captures, query params, request bodies,
@@ -63,97 +60,134 @@ serversEqualProperty api mgr burl1 burl2 = property $ ShouldMatch c1 c2
6360-- run.
6461--
6562-- Evidently, if the behaviour of the server is expected to be
66−-- non-deterministic, this function may produce spurious failures.
67−serversEqual :: (HasClient a, Testable (ShouldMatch (Client a)))
68− => Proxy a -> BaseUrl -> BaseUrl -> Int -> Expectation
69−serversEqual api burl1 burl2 tries = do
70− mgr <- managerWithStoredReq
71− let args = stdArgs { chatty = False, maxSuccess = tries }
72− res <- quickCheckWithResult args $ serversEqualProperty api mgr burl1 burl2
73− case res of
74− Success _ _ _ -> return ()
75− _ -> prettyErr >>= expectationFailure
76−
77−
78−serverSatisfiesProperty :: (HasClient a, Testable (ShouldSatisfy filt exp (Client a)))
79− => Proxy a -> Manager -> BaseUrl -> Predicates filt -> Predicates exp -> Property
80−serverSatisfiesProperty api mgr burl filters expect = do
81− property $ ShouldSatisfy (client api burl mgr) filters expect
82−
83−-- | Check that a server's responses satisfies certain properties.
84−serverSatisfies :: (HasClient a, Testable (ShouldSatisfy filt exp (Client a)))
85− => Proxy a -> BaseUrl -> Predicates filt -> Predicates exp
86− -> Int -> Expectation
87−serverSatisfies api burl filters expect tries = do
88− mgr <- managerWithStoredReq
89− let args = stdArgs { chatty = False, maxSuccess = tries }
90− res <- quickCheckWithResult args $ serverSatisfiesProperty api mgr burl filters expect
91− case res of
92− Success _ _ _ -> return ()
93− GaveUp n _ _ -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
94− _ -> prettyErr >>= expectationFailure
95−
96−-- | Check that the two servers running under the provided @BaseUrl@s do not
97−-- behave identically.
63+-- non-deterministic, this function may produce spurious failures
9864--
99−-- As with @serversEqualProperty@, non-determinism in the servers will likely
100−-- result in failures that may not be significant.
101−serversUnequal :: (HasClient a, Testable (ShouldMatch (Client a)))
102− => Proxy a -> BaseUrl -> BaseUrl -> Int -> Expectation
103−serversUnequal api burl1 burl2 tries = do
104− mgr <- managerWithStoredReq
105− let args = stdArgs { chatty = False, maxSuccess = tries }
106− res <- quickCheckWithResult args $ serversEqualProperty api mgr burl1 burl2
107− case res of
108− Success _ _ _ -> prettyErr >>= expectationFailure
109− _ -> return ()
65+-- Note that only valid requests are generated and tested. As an example of why
66+-- this matters, let's say your API specifies that a particular endpoint can
67+-- only generate @JSON@. @serversEqual@ will then not generate any requests
68+-- with an @Accept@ header _other_ than @application/json@. It may therefore
69+-- fail to notice that one application, when the request has @Accept:
70+-- text/html@, returns a @406 Not Acceptable@ HTTP response, and another
71+-- returns a @200 Success@, but with @application/json@ as the content-type.
72+--
73+-- The fact that only valid requests are tested also means that no endpoints
74+-- not listed in the API type are tested.
75+--
76+-- /Since 0.0.0.0/
77+serversEqual :: HasGenRequest a =>
78+ Proxy a -> BaseUrl -> BaseUrl -> Args -> ResponseEquality LBS.ByteString -> Expectation
79+serversEqual api burl1 burl2 args req = do
80+ let reqs = (\f -> (f burl1, f burl2)) <$> runGenRequest api
81+ -- This MVar stuff is clunky! But there doesn't seem to be an easy way to
82+ -- return results when a test fails, since an exception is throw.
83+ deetsMVar <- newEmptyMVar
84+ r <- quickCheckWithResult args { chatty = False } $ monadicIO $ forAllM reqs $ \(req1, req2) -> do
85+ resp1 <- run $ C.httpLbs (noCheckStatus req1) defManager
86+ resp2 <- run $ C.httpLbs (noCheckStatus req2) defManager
87+ unless (getResponseEquality req resp1 resp2) $ do
88+ monitor (counterexample "hi" )
89+ _ <- run $ tryPutMVar deetsMVar $ ServerEqualityFailure req1 resp1 resp2
90+ assert False
91+ case r of
92+ Success {} -> return ()
93+ Failure{..} -> do
94+ mx <- tryReadMVar deetsMVar
95+ case mx of
96+ Just x ->
97+ expectationFailure $ "Failed:\n" ++ show x
98+ Nothing ->
99+ expectationFailure $ "We failed to record a reason for failure: " <> show r
100+ GaveUp { numTests = n } -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
101+ NoExpectedFailure {} -> expectationFailure "No expected failure"
102+#if MIN_VERSION_QuickCheck(2,12,0)
103+#else
104+ InsufficientCoverage {} -> expectationFailure "Insufficient coverage"
105+#endif
110106
111−serverDoesntSatisfy :: (HasClient a, Testable (ShouldSatisfy filt exp (Client a)))
112− => Proxy a -> BaseUrl -> Predicates filt -> Predicates exp
113− -> Int -> Expectation
114−serverDoesntSatisfy api burl filters expect tries = do
115− mgr <- managerWithStoredReq
116− let args = stdArgs { chatty = False, maxSuccess = tries }
117− res <- quickCheckWithResult args $ serverSatisfiesProperty api mgr burl filters expect
118− case res of
119− Success _ _ _ -> prettyErr >>= expectationFailure
120− _ -> return ()
107+-- | Check that a server satisfies the set of properties specified.
108+--
109+-- Note that, rather than having separate tests for each property you'd like to
110+-- test, you should generally prefer to combine all properties into a single
111+-- test. This enables a more parsimonious generation of requests and responses
112+-- with the same testing depth.
113+--
114+-- Example usage:
115+--
116+-- > goodAPISpec = describe "my server" $ do
117+-- >
118+-- > it "follows best practices" $ do
119+-- > withServantServer api server $ \burl ->
120+-- > serverSatisfies api burl stdArgs (not500
121+-- > <%> onlyJsonObjects
122+-- > <%> notAllowedContainsAllowHeader
123+-- > <%> mempty)
124+--
125+-- /Since 0.0.0.0/
126+serverSatisfies :: (HasGenRequest a) =>
127+ Proxy a -> BaseUrl -> Args -> Predicates -> Expectation
128+serverSatisfies api = serverSatisfiesMgr api defManager
121129
122−-- | Benchmarks a server with arbitrary requests using 'wrk'.
130+-- | Check that a server satisfies the set of properties specified, and
131+-- accept a 'Manager' for running the HTTP requests through.
123132--
124−-- When using this, you should compile your program with '-threaded'.
125−-- Moreover, 'wrk' must be in the @$PATH@.
133+-- See 'serverSatisfies' for more details.
126134--
127−-- Note that this function is still very experimental, and it's behaviour will
128−-- likely change.
129−serverBenchmark ::
130− (HasClient a , Testable (ShouldSatisfy '[] '[Either ServantError Void] (Client a)))
131− => Proxy a -> BaseUrl -> BenchOptions -> IO ()
132−serverBenchmark api burl opts = replicateM_ (noOfTests opts) go
133− where
134− go = do
135− let alwaysTrue = addLeftPredicate (const True) emptyPredicates
136− serverSatisfies api burl emptyPredicates alwaysTrue 1
137− Just (r, _) <- readMVar currentReq
138− withSystemTempFile "wrkscript.lua" $ \f h -> do
139− let url = show $ getUri r
140− s = mkScript $ reqToWrk r
141− c = "wrk -c" ++ show (connections opts)
142− ++ " -d" ++ show (duration opts) ++ "s "
143− ++ " -t" ++ show (threads opts)
144− ++ " -s \"" ++ f ++ "\" "
145− ++ " --latency "
146− ++ url
147− hPutStrLn h s
148− hFlush h
149− callCommand c
150− -- While running wrk and the server on the same machine make the
151− -- results much less meaningful, this ameliorates the situation
152− -- somewhat.
153− performGC
154− threadDelay 1000
135+-- @since 0.0.7.2
136+serverSatisfiesMgr :: (HasGenRequest a) =>
137+ Proxy a -> C.Manager -> BaseUrl -> Args -> Predicates -> Expectation
138+serverSatisfiesMgr api manager burl args preds = do
139+ let reqs = ($ burl) <$> runGenRequest api
140+ deetsMVar <- newEmptyMVar
141+ r <- quickCheckWithResult args { chatty = False } $ monadicIO $ forAllM reqs $ \req -> do
142+ v <- run $ finishPredicates preds (noCheckStatus req) manager
143+ _ <- run $ tryPutMVar deetsMVar v
144+ case v of
145+ Just _ -> assert False
146+ _ -> return ()
147+ case r of
148+ Success {} -> return ()
149+ Failure {..} -> do
150+ mx <- tryReadMVar deetsMVar
151+ case mx of
152+ Just x ->
153+ expectationFailure $ "Failed:\n" ++ show x
154+ Nothing ->
155+ expectationFailure $ "We failed to record a reason for failure: " <> show r
156+ GaveUp { numTests = n } -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
157+ NoExpectedFailure {} -> expectationFailure $ "No expected failure"
158+#if MIN_VERSION_QuickCheck(2,12,0)
159+#else
160+ InsufficientCoverage {} -> expectationFailure "Insufficient coverage"
161+#endif
162+
163+serverDoesntSatisfy :: (HasGenRequest a) =>
164+ Proxy a -> BaseUrl -> Args -> Predicates -> Expectation
165+serverDoesntSatisfy api = serverDoesntSatisfyMgr api defManager
166+
167+serverDoesntSatisfyMgr :: (HasGenRequest a) =>
168+ Proxy a -> C.Manager -> BaseUrl -> Args -> Predicates -> Expectation
169+serverDoesntSatisfyMgr api manager burl args preds = do
170+ let reqs = ($ burl) <$> runGenRequest api
171+ r <- quickCheckWithResult args $ monadicIO $ forAllM reqs $ \req -> do
172+ v <- run $ finishPredicates preds (noCheckStatus req) manager
173+ assert $ not $ null v
174+ case r of
175+ Success {} -> return ()
176+ GaveUp { numTests = n } -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
177+ Failure { output = m } -> expectationFailure $ "Failed:\n" ++ show m
178+ NoExpectedFailure {} -> expectationFailure $ "No expected failure"
179+#if MIN_VERSION_QuickCheck(2,12,0)
180+#else
181+ InsufficientCoverage {} -> expectationFailure "Insufficient coverage"
182+#endif
183+
184+noCheckStatus :: C.Request -> C.Request
185+#if MIN_VERSION_http_client(0,5,0)
186+noCheckStatus = id
187+#else
188+noCheckStatus r = r { C.checkStatus = \_ _ _ -> Nothing}
189+#endif
155190
156−managerWithStoredReq :: IO Manager
157−managerWithStoredReq = newManager defaultManagerSettings { managerModifyRequest = go }
158− where go req = modifyMVar_ currentReq (addReq req) >> return req
159− addReq req _ = return $ Just (req, "")
191+defManager :: C.Manager
192+defManager = unsafePerformIO $ C.newManager C.defaultManagerSettings
193+{-# NOINLINE defManager #-}
src/Servant/QuickCheck/Internal/Testable.hsdeleted−96
@@ -1,96 +0,0 @@
1−-- | This module contains QuickCheck-related logic.
2−module Servant.QuickCheck.Internal.Testable where
3−
4−import Control.Concurrent.MVar (MVar, modifyMVar_, newMVar, readMVar)
5−import Control.Monad.Except (runExceptT)
6−import GHC.Generics (Generic)
7−import Network.HTTP.Client (Request, RequestBody (..),
8− requestBody)
9−import Servant.API ((:<|>)(..))
10−import Servant.Client (ServantError (..), ClientM)
11−import System.IO.Unsafe (unsafePerformIO)
12−import Test.QuickCheck (Arbitrary (..), discard)
13−import Test.QuickCheck.Property (Testable (..), forAllShrink,
14− ioProperty, (.&.))
15−
16−import Servant.QuickCheck.Internal.Predicates
17−
18−
19−-- * ShouldMatch
20−
21−-- | Two corresponding client functions. Used for checking that APIs match.
22−data ShouldMatch a = ShouldMatch a a
23− deriving (Eq, Show, Read, Generic)
24−
25−instance (Show a, Eq a) => Testable (ShouldMatch (ClientM a)) where
26− property (ShouldMatch e1 e2) = ioProperty $ do
27− e1' <- runExceptT e1
28− e2' <- runExceptT e2
29− modifyMVar_ currentReq $ \x -> case x of
30− Nothing -> error "impossible"
31− Just (x', _) -> return $ Just (x', "LHS:\n" ++ show e1'
32− ++ "\nRHS:\n" ++ show e2')
33− case (e1', e2') of
34− (Right v1, Right v2) -> return $ v1 == v2
35− (Left (FailureResponse a1 b1 c1), Left (FailureResponse a2 b2 c2)) ->
36− return $ a1 == a2 && b1 == b2 && c1 == c2
37− (err1, err2) -> error $ "Exception response:"
38− ++ "\nLHS:\n" ++ show err1
39− ++ "\nRHS:\n" ++ show err2
40−
41−instance (Arbitrary a, Show a, Testable (ShouldMatch b))
42− => Testable (ShouldMatch (a -> b)) where
43− property (ShouldMatch f1 f2) = forAllShrink arbitrary shrink go
44− where go x = ShouldMatch (f1 x) (f2 x)
45−
46−instance (Testable (ShouldMatch a), Testable (ShouldMatch b))
47− => Testable (ShouldMatch (a :<|> b)) where
48− property (ShouldMatch (a1 :<|> b1) (a2 :<|> b2))
49− = property (ShouldMatch a1 a2) .&. property (ShouldMatch b1 b2)
50−
51−-- * ShouldSatisfy
52−
53−data ShouldSatisfy filter expect a = ShouldSatisfy
54− { ssVal :: a
55− , ssFilter :: Predicates filter
56− , ssExpect :: Predicates expect
57− } deriving (Functor)
58−
59−instance (Show a, Eq a, HasPredicate expect (Either ServantError a))
60− => Testable (ShouldSatisfy filter expect (ClientM a)) where
61− property (ShouldSatisfy a _ e) = ioProperty $ do
62− a' <- runExceptT a
63− modifyMVar_ currentReq $ \x -> case x of
64− Nothing -> error "impossible"
65− Just (x', _) -> return $ Just (x', show a')
66− return $ getPredicate e a'
67−
68−instance ( Arbitrary a, Show a, Testable (ShouldSatisfy filter expect b)
69− , HasPredicate filter a)
70− => Testable (ShouldSatisfy filter expect (a -> b)) where
71− property (ShouldSatisfy g f e) = forAllShrink arbitrary shrink go
72− where go x | getPredicate f x = ShouldSatisfy (g x) f e
73− | otherwise = discard
74−
75−instance ( Testable (ShouldSatisfy filter expect a)
76− , Testable (ShouldSatisfy filter expect b))
77− => Testable (ShouldSatisfy filter expect (a :<|> b)) where
78− property (ShouldSatisfy (a :<|> b) f e)
79− = property (ShouldSatisfy a f e) .&. property (ShouldSatisfy b f e)
80−
81−-- * Utils
82−
83−-- Used to store the current request and response so that in case of failure we
84−-- have the failing test in a user-friendly form.
85−currentReq :: MVar (Maybe (Request, String))
86−currentReq = unsafePerformIO $ newMVar Nothing
87−{-# NOINLINE currentReq #-}
88−
89−prettyErr :: IO String
90−prettyErr = do
91− Just (req, resp) <- readMVar currentReq
92− return $ show req ++ "Body:\n" ++ showReqBody (requestBody req)
93− ++ "\n\nResponse:\n" ++ resp
94− where
95− showReqBody (RequestBodyLBS x) = show x
96− showReqBody _ = error "expecting RequestBodyLBS"
stack.yaml+5−31
@@ -1,36 +1,10 @@
1−# Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2)
2−resolver: nightly-2016-04-20
3−
4−# Local packages, usually specified by relative directory name
1+resolver: nightly-2018-09-03
52packages:
63- '.'
7−- 'doc'
8−# Packages to be pulled from upstream that are not in the resolver (e.g., acme-missiles-0.3)
94extra-deps:
10−- servant-0.7
11−- servant-client-0.7
12−- servant-server-0.7
13−
14−# Override default flag values for local packages and extra-deps
5+- hspec-discover-2.5.6
6+- hspec-core-2.5.6
7+- hspec-2.5.6
8+- QuickCheck-2.12
159flags: {}
16−
17−# Extra package databases containing global packages
1810extra-package-dbs: []
19−
20−# Control whether we use the GHC we find on the path
21−# system-ghc: true
22−
23−# Require a specific version of stack, using version ranges
24−# require-stack-version: -any # Default
25−# require-stack-version: >= 1.0.0
26−
27−# Override the architecture used by stack, especially useful on Windows
28−# arch: i386
29−# arch: x86_64
30−
31−# Extra directories used by stack for building
32−# extra-include-dirs: [/path/to/dir]
33−# extra-lib-dirs: [/path/to/dir]
34−
35−# Allow a newer minor version of GHC than the snapshot specifies
36−# compiler-check: newer-minor
test/Doctest.hsdeleted−44
@@ -1,44 +0,0 @@
1−module Main where
2−
3−import Data.List (isPrefixOf)
4−import System.Directory
5−import System.FilePath
6−import System.FilePath.Find
7−import Test.DocTest
8−
9−main :: IO ()
10−main = do
11− files <- find always (extension ==? ".hs") "src"
12− mCabalMacrosFile <- getCabalMacrosFile
13− doctest $ "-isrc" : "-Iinclude" :
14− (maybe [] (\ f -> ["-optP-include", "-optP" ++ f]) mCabalMacrosFile) ++
15− "-XOverloadedStrings" :
16− "-XDeriveFunctor" :
17− "-XFlexibleInstances" :
18− "-XFlexibleContexts" :
19− "-XMultiParamTypeClasses" :
20− "-XDataKinds" :
21− "-XTypeOperators" :
22− "-XGADTs" :
23− files
24−
25−getCabalMacrosFile :: IO (Maybe FilePath)
26−getCabalMacrosFile = do
27− exists <- doesDirectoryExist "dist"
28− if exists
29− then do
30− contents <- getDirectoryContents "dist"
31− let rest = "build" </> "autogen" </> "cabal_macros.h"
32− whenExists $ case filter ("dist-sandbox-" `isPrefixOf`) contents of
33− [x] -> "dist" </> x </> rest
34− [] -> "dist" </> rest
35− xs -> error $ "ran doctests with multiple dist/dist-sandbox-xxxxx's: \n"
36− ++ show xs ++ "\nTry cabal clean"
37− else return Nothing
38− where
39− whenExists :: FilePath -> IO (Maybe FilePath)
40− whenExists file = do
41− exists <- doesFileExist file
42− return $ if exists
43− then Just file
44− else Nothing
test/Servant/CoMock/InternalSpec.hsdeleted−170
@@ -1,170 +0,0 @@
1−{-# LANGUAGE CPP #-}
2−{-# LANGUAGE ScopedTypeVariables #-}
3−module Servant.CoMock.InternalSpec (spec) where
4−
5−import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
6−import Control.Monad.IO.Class (liftIO)
7−import Data.Proxy
8−import Servant
9−import Test.Hspec
10−
11−import Servant.CoMock.Internal
12−
13−spec :: Spec
14−spec = do
15− serversEqualSpec
16− serverSatisfiesSpec
17− serverBenchmarkSpec
18−
19−
20−serversEqualSpec :: Spec
21−serversEqualSpec = describe "serversEqual" $ do
22−
23− context "servers without function types" $ do
24−
25− it "considers equal servers equal" $ do
26− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
27− serversEqual onlyReturnAPI burl burl noOfTestCases
28−
29− it "considers unequal servers unequal" $ do
30− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl1 ->
31− withServantServer onlyReturnAPI onlyReturnAPIServer' $ \burl2 ->
32− serversUnequal onlyReturnAPI burl1 burl2 noOfTestCases
33−
34−
35− context "servers with function types" $ do
36−
37− it "considers equal servers equal" $ do
38− withServantServer functionAPI functionAPIServer $ \burl ->
39− serversEqual functionAPI burl burl noOfTestCases
40−
41− it "considers unequal servers unequal" $ do
42− withServantServer functionAPI functionAPIServer $ \burl1 ->
43− withServantServer functionAPI functionAPIServer' $ \burl2 ->
44− serversUnequal functionAPI burl1 burl2 noOfTestCases
45−
46−
47− context "stateful servers" $ do
48−
49− it "considers equal servers equal" $ do
50− withServantServer statefulAPI statefulAPIServer $ \burl1 ->
51− withServantServer statefulAPI statefulAPIServer $ \burl2 ->
52− serversEqual statefulAPI burl1 burl2 noOfTestCases
53−
54−
55−serverSatisfiesSpec :: Spec
56−serverSatisfiesSpec = describe "serverSatisfies" $ do
57−
58− it "passes true predicates" $ do
59− let e = addRightPredicate (== (5 :: Int)) emptyPredicates
60− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
61− serverSatisfies onlyReturnAPI burl emptyPredicates e noOfTestCases
62−
63− it "fails false predicates" $ do
64− let e = addRightPredicate (== (4 :: Int)) emptyPredicates
65− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
66− serverDoesntSatisfy onlyReturnAPI burl emptyPredicates e noOfTestCases
67−
68− it "allows filtering" $ do
69− let f = addPredicate (\(x :: String) -> length x > 2) emptyPredicates
70− e = addRightPredicate (\(x :: Int) -> x > 2) emptyPredicates
71− e' = addRightPredicate (\(x :: Int) -> x < 2) emptyPredicates
72− withServantServer functionAPI functionAPIServer $ \burl -> do
73− serverSatisfies functionAPI burl f e noOfTestCases
74− serverDoesntSatisfy functionAPI burl f e' noOfTestCases
75−
76− it "allows polymorphic predicates" $ do
77− let p1 x = length (show x) < 100000
78− p2 x = length (show x) < 1
79− e1 = addPolyPredicate (Proxy :: Proxy Show) p1 emptyPredicates
80− e2 = addPolyPredicate (Proxy :: Proxy Show) p2 emptyPredicates
81− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl -> do
82− serverSatisfies onlyReturnAPI burl emptyPredicates e1 noOfTestCases
83− serverDoesntSatisfy onlyReturnAPI burl emptyPredicates e2 noOfTestCases
84−
85−
86− context "never500s" $ do
87−
88− it "is true for servers that don't return 500s" $ do
89− withServantServer functionAPI functionAPIServer $ \burl ->
90− serverSatisfies functionAPI burl emptyPredicates never500s noOfTestCases
91−
92− it "is false for servers that return 500s" $ do
93− withServantServer onlyReturnAPI onlyReturnAPIServer'' $ \burl ->
94− serverDoesntSatisfy onlyReturnAPI burl emptyPredicates never500s noOfTestCases
95−
96− context "onlyJsonObjects" $ do
97−
98− it "is false for servers that return top-level literals" $ do
99− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
100− serverDoesntSatisfy onlyReturnAPI burl emptyPredicates onlyJsonObjects noOfTestCases
101−
102−
103−serverBenchmarkSpec :: Spec
104−serverBenchmarkSpec = describe "serverBenchmark" $ do
105−
106− it "works" $ do
107− withServantServer onlyReturnAPI onlyReturnAPIServer $ \burl ->
108− serverBenchmark onlyReturnAPI burl defaultBenchOptions
109−
110−------------------------------------------------------------------------------
111−-- APIs
112−------------------------------------------------------------------------------
113−
114−-- * OnlyReturn
115−
116−type OnlyReturnAPI = Get '[JSON] Int
117− :<|> Post '[JSON] String
118−
119−onlyReturnAPI :: Proxy OnlyReturnAPI
120−onlyReturnAPI = Proxy
121−
122−onlyReturnAPIServer :: IO (Server OnlyReturnAPI)
123−onlyReturnAPIServer = return $ return 5 :<|> return "hi"
124−
125−onlyReturnAPIServer' :: IO (Server OnlyReturnAPI)
126−onlyReturnAPIServer' = return $ return 5 :<|> return "hia"
127−
128−onlyReturnAPIServer'' :: IO (Server OnlyReturnAPI)
129−onlyReturnAPIServer'' = return $ error "err" :<|> return "hia"
130−
131−-- * Function
132−
133−type FunctionAPI = ReqBody '[JSON] String :> Post '[JSON] Int
134− :<|> Header "X-abool" Bool :> Get '[JSON] (Maybe Bool)
135−
136−functionAPI :: Proxy FunctionAPI
137−functionAPI = Proxy
138−
139−functionAPIServer :: IO (Server FunctionAPI)
140−functionAPIServer = return $ return . length :<|> return
141−
142−functionAPIServer' :: IO (Server FunctionAPI)
143−functionAPIServer'
144− = return $ (\x -> return $ length x - 1) :<|> \x -> return (not <$> x)
145−
146−-- * Stateful
147−
148−type StatefulAPI = ReqBody '[JSON] String :> Post '[JSON] String
149− :<|> Get '[JSON] Int
150−
151−statefulAPI :: Proxy StatefulAPI
152−statefulAPI = Proxy
153−
154−statefulAPIServer :: IO (Server StatefulAPI)
155−statefulAPIServer = do
156− mvar <- newMVar ""
157− return $ (\x -> liftIO $ swapMVar mvar x)
158− :<|> (liftIO $ readMVar mvar >>= return . length)
159−
160−
161−------------------------------------------------------------------------------
162−-- Utils
163−------------------------------------------------------------------------------
164−
165−noOfTestCases :: Int
166−#if LONG_TESTS
167−noOfTestCases = 20000
168−#else
169−noOfTestCases = 500
170−#endif
test/Servant/QuickCheck/InternalSpec.hsnew file+371
@@ -0,0 +1,371 @@
1+{-# LANGUAGE CPP #-}
2+module Servant.QuickCheck.InternalSpec (spec) where
3+
4+
5+import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
6+import Control.Exception (SomeException)
7+import Control.Monad (replicateM)
8+import Control.Monad.IO.Class (liftIO)
9+import qualified Data.ByteString as BS
10+import qualified Data.ByteString.Char8 as C
11+import Data.Maybe (fromJust)
12+import Network.HTTP.Client (path, queryString)
13+import Prelude.Compat
14+import Servant
15+import Servant.HTML.Blaze (HTML)
16+import qualified Text.Blaze.Html as Blaze
17+import qualified Text.Blaze.Html5 as Blaze5
18+import Test.Hspec (Spec, context, describe, it, shouldBe,
19+ shouldContain)
20+import Test.Hspec.Core.Spec (Arg, Example, Result (..), ResultStatus (..),
21+ defaultParams, safeEvaluateExample)
22+import Test.QuickCheck.Gen (generate, unGen)
23+import Test.QuickCheck.Random (mkQCGen)
24+
25+
26+import Servant.Test.ComprehensiveAPI (comprehensiveAPIWithoutStreamingOrRaw)
27+
28+import Servant.QuickCheck
29+import Servant.QuickCheck.Internal (genRequest, runGenRequest,
30+ serverDoesntSatisfy)
31+
32+spec :: Spec
33+spec = do
34+ serversEqualSpec
35+ serverSatisfiesSpec
36+ isComprehensiveSpec
37+ onlyJsonObjectSpec
38+ notLongerThanSpec
39+ queryParamsSpec
40+ queryFlagsSpec
41+ deepPathSpec
42+ htmlDocTypesSpec
43+ unbiasedGenerationSpec
44+
45+serversEqualSpec :: Spec
46+serversEqualSpec = describe "serversEqual" $ do
47+
48+ it "considers equal servers equal" $ do
49+ withServantServerAndContext api ctx server $ \burl1 ->
50+ withServantServerAndContext api ctx server $ \burl2 -> do
51+ serversEqual api burl1 burl2 args bodyEquality
52+
53+ context "when servers are not equal" $ do
54+ it "provides the failing responses in the error message" $ do
55+ FailedWith err <- withServantServer api2 server2 $ \burl1 ->
56+ withServantServer api2 server3 $ \burl2 -> do
57+ evalExample $ serversEqual api2 burl1 burl2 args bodyEquality
58+ show err `shouldContain` "Server equality failed"
59+ show err `shouldContain` "Body: 1"
60+ show err `shouldContain` "Body: 2"
61+ show err `shouldContain` "Path: /failplz"
62+
63+ context "when JSON is equal but looks a bit different as a ByteString" $ do
64+
65+ it "sanity check: different whitespace same JSON objects bodyEquality fails" $ do
66+ FailedWith err <- withServantServer jsonApi jsonServer1 $ \burl1 ->
67+ withServantServer jsonApi jsonServer2 $ \burl2 -> do
68+ evalExample $ serversEqual jsonApi burl1 burl2 args bodyEquality
69+ show err `shouldContain` "Server equality failed"
70+
71+ it "jsonEquality considers equal JSON apis equal regardless of key ordering or whitespace" $ do
72+ withServantServerAndContext jsonApi ctx jsonServer1 $ \burl1 ->
73+ withServantServerAndContext jsonApi ctx jsonServer2 $ \burl2 ->
74+ serversEqual jsonApi burl1 burl2 args jsonEquality
75+
76+ it "sees when JSON apis are not equal because any value is different" $ do
77+ FailedWith err <- withServantServer jsonApi jsonServer2 $ \burl1 ->
78+ withServantServer jsonApi jsonServer3 $ \burl2 -> do
79+ evalExample $ serversEqual jsonApi burl1 burl2 args jsonEquality
80+ show err `shouldContain` "Server equality failed"
81+ show err `shouldContain` "Path: /jsonComparison"
82+
83+ it "sees when JSON apis are not equal due to different keys but same values" $ do
84+ FailedWith err <- withServantServer jsonApi jsonServer2 $ \burl1 ->
85+ withServantServer jsonApi jsonServer4 $ \burl2 -> do
86+ evalExample $ serversEqual jsonApi burl1 burl2 args jsonEquality
87+ show err `shouldContain` "Server equality failed"
88+ show err `shouldContain` "Path: /jsonComparison"
89+
90+
91+serverSatisfiesSpec :: Spec
92+serverSatisfiesSpec = describe "serverSatisfies" $ do
93+
94+ it "succeeds for true predicates" $ do
95+ withServantServerAndContext api ctx server $ \burl ->
96+ serverSatisfies api burl args (unauthorizedContainsWWWAuthenticate
97+ <%> not500
98+ <%> mempty)
99+
100+ it "fails for false predicates" $ do
101+ withServantServerAndContext api ctx server $ \burl -> do
102+ serverDoesntSatisfy api burl args (onlyJsonObjects
103+ <%> getsHaveCacheControlHeader
104+ <%> headsHaveCacheControlHeader
105+ <%> notAllowedContainsAllowHeader
106+ <%> mempty)
107+
108+ context "when predicates are false" $ do
109+
110+ it "fails with informative error messages" $ do
111+ FailedWith err <- withServantServerAndContext api ctx server $ \burl -> do
112+ evalExample $ serverSatisfies api burl args (notAllowedContainsAllowHeader <%> mempty)
113+ show err `shouldContain` "notAllowedContainsAllowHeader"
114+ show err `shouldContain` "Headers"
115+ show err `shouldContain` "Body"
116+
117+
118+onlyJsonObjectSpec :: Spec
119+onlyJsonObjectSpec = describe "onlyJsonObjects" $ do
120+
121+ it "fails correctly" $ do
122+ FailedWith err <- withServantServerAndContext api ctx server $ \burl -> do
123+ evalExample $ serverSatisfies (Proxy :: Proxy (Get '[JSON] Int)) burl args
124+ (onlyJsonObjects <%> mempty)
125+ show err `shouldContain` "onlyJsonObjects"
126+
127+ it "accepts non-JSON endpoints" $ do
128+ withServantServerAndContext octetAPI ctx serverOctetAPI $ \burl ->
129+ serverSatisfies octetAPI burl args (onlyJsonObjects <%> mempty)
130+
131+ it "does not fail when there is no content-type" $ do
132+ withServantServerAndContext api2 ctx serverFailing $ \burl ->
133+ serverSatisfies api2 burl args (onlyJsonObjects <%> mempty)
134+
135+notLongerThanSpec :: Spec
136+notLongerThanSpec = describe "notLongerThan" $ do
137+
138+ it "fails correctly" $ do
139+ FailedWith err <- withServantServerAndContext api ctx server $ \burl -> do
140+ evalExample $ serverSatisfies (Proxy :: Proxy (Get '[JSON] Int)) burl args
141+ (notLongerThan 1 <%> mempty)
142+ show err `shouldContain` "notLongerThan"
143+
144+ it "succeeds correctly" $ do
145+ withServantServerAndContext api ctx server $ \burl ->
146+ serverSatisfies api burl args (notLongerThan 1000000000000 <%> mempty)
147+
148+isComprehensiveSpec :: Spec
149+isComprehensiveSpec = describe "HasGenRequest" $ do
150+
151+ it "has instances for all 'servant' combinators" $ do
152+ let _g = genRequest comprehensiveAPIWithoutStreamingOrRaw
153+ True `shouldBe` True -- This is a type-level check
154+
155+deepPathSpec :: Spec
156+deepPathSpec = describe "Path components" $ do
157+
158+ it "are separated by slashes, without a trailing slash" $ do
159+ let rng = mkQCGen 0
160+ burl = BaseUrl Http "localhost" 80 ""
161+ gen = runGenRequest deepAPI
162+ req = (unGen gen rng 0) burl
163+ path req `shouldBe` ("/one/two/three")
164+
165+
166+queryParamsSpec :: Spec
167+queryParamsSpec = describe "QueryParams" $ do
168+
169+ it "reduce to an HTTP query string correctly" $ do
170+ let rng = mkQCGen 0
171+ burl = BaseUrl Http "localhost" 80 ""
172+ gen = runGenRequest paramsAPI
173+ req = (unGen gen rng 0) burl
174+ qs = C.unpack $ queryString req
175+ qs `shouldBe` "one=_&two=_"
176+
177+queryFlagsSpec :: Spec
178+queryFlagsSpec = describe "QueryFlags" $ do
179+
180+ it "reduce to an HTTP query string correctly" $ do
181+ let rng = mkQCGen 0
182+ burl = BaseUrl Http "localhost" 80 ""
183+ gen = runGenRequest flagsAPI
184+ req = (unGen gen rng 0) burl
185+ qs = C.unpack $ queryString req
186+ qs `shouldBe` "one&two"
187+
188+htmlDocTypesSpec :: Spec
189+htmlDocTypesSpec = describe "HtmlDocTypes" $ do
190+
191+ it "fails HTML without doctype correctly" $ do
192+ err <- withServantServerAndContext docTypeApi ctx noDocTypeServer $ \burl -> do
193+ evalExample $ serverSatisfies docTypeApi burl args
194+ (htmlIncludesDoctype <%> mempty)
195+ show err `shouldContain` "htmlIncludesDoctype"
196+
197+ it "passes HTML with a doctype at start" $ do
198+ withServantServerAndContext docTypeApi ctx docTypeServer $ \burl ->
199+ serverSatisfies docTypeApi burl args (htmlIncludesDoctype <%> mempty)
200+
201+ it "accepts json endpoints and passes over them in silence" $ do
202+ withServantServerAndContext api ctx server $ \burl -> do
203+ serverSatisfies (Proxy :: Proxy (Get '[JSON] Int)) burl args
204+ (htmlIncludesDoctype <%> mempty)
205+
206+
207+makeRandomRequest :: Proxy LargeAPI -> BaseUrl -> IO Integer
208+makeRandomRequest large burl = do
209+ req <- generate $ runGenRequest large
210+ pure $ fst . fromJust . C.readInteger . C.drop 1 . path $ req burl
211+
212+
213+unbiasedGenerationSpec :: Spec
214+unbiasedGenerationSpec = describe "Unbiased Generation of requests" $
215+
216+ it "frequency paired with generated endpoint should be more randomly distributed" $ do
217+ let burl = BaseUrl Http "localhost" 80 ""
218+ let runs = 10000 :: Double
219+ someRequests <- replicateM 10000 (makeRandomRequest largeApi burl)
220+ let mean = (sum $ map fromIntegral someRequests) / runs
221+ let variancer x = let ix = fromIntegral x in (ix - mean) * (ix - mean)
222+ let variance = (sum $ map variancer someRequests) / runs - 1
223+ -- mean should be around 8.5. If this fails, we likely need more runs (or there's a bug!)
224+ mean > 8 `shouldBe` True
225+ mean < 9 `shouldBe` True
226+ -- Std dev is likely around 4. Variance is probably greater than 20.
227+ variance > 19.5 `shouldBe` True
228+
229+------------------------------------------------------------------------------
230+-- APIs
231+------------------------------------------------------------------------------
232+
233+type API = ReqBody '[JSON] String :> Post '[JSON] String
234+ :<|> Get '[JSON] Int
235+ :<|> BasicAuth "some-realm" () :> Get '[JSON] ()
236+
237+api :: Proxy API
238+api = Proxy
239+
240+type ParamsAPI = QueryParam "one" () :> QueryParam "two" () :> Get '[JSON] ()
241+
242+paramsAPI :: Proxy ParamsAPI
243+paramsAPI = Proxy
244+
245+type FlagsAPI = QueryFlag "one" :> QueryFlag "two" :> Get '[JSON] ()
246+
247+flagsAPI :: Proxy FlagsAPI
248+flagsAPI = Proxy
249+
250+
251+server :: IO (Server API)
252+server = do
253+ mvar <- newMVar ""
254+ return $ (\x -> liftIO $ swapMVar mvar x)
255+ :<|> (liftIO $ readMVar mvar >>= return . length)
256+ :<|> (const $ return ())
257+
258+
259+type API2 = "failplz" :> Get '[JSON] Int
260+
261+api2 :: Proxy API2
262+api2 = Proxy
263+
264+type DeepAPI = "one" :> "two" :> "three":> Get '[JSON] ()
265+
266+deepAPI :: Proxy DeepAPI
267+deepAPI = Proxy
268+
269+
270+server2 :: IO (Server API2)
271+server2 = return $ return 1
272+
273+server3 :: IO (Server API2)
274+server3 = return $ return 2
275+
276+serverFailing :: IO (Server API2)
277+serverFailing = return . throwError $ err405
278+
279+-- With Doctypes
280+type HtmlDoctype = Get '[HTML] Blaze.Html
281+
282+docTypeApi :: Proxy HtmlDoctype
283+docTypeApi = Proxy
284+
285+docTypeServer :: IO (Server HtmlDoctype)
286+docTypeServer = pure $ pure $ Blaze5.docTypeHtml $ Blaze5.span "Hello Test!"
287+
288+noDocTypeServer :: IO (Server HtmlDoctype)
289+noDocTypeServer = pure $ pure $ Blaze.text "Hello Test!"
290+
291+
292+-- Api for unbiased generation of requests tests
293+largeApi :: Proxy LargeAPI
294+largeApi = Proxy
295+
296+type LargeAPI
297+ = "1" :> Get '[JSON] Int
298+ :<|> "2" :> Get '[JSON] Int
299+ :<|> "3" :> Get '[JSON] Int
300+ :<|> "4" :> Get '[JSON] Int
301+ :<|> "5" :> Get '[JSON] Int
302+ :<|> "6" :> Get '[JSON] Int
303+ :<|> "7" :> Get '[JSON] Int
304+ :<|> "8" :> Get '[JSON] Int
305+ :<|> "9" :> Get '[JSON] Int
306+ :<|> "10" :> Get '[JSON] Int
307+ :<|> "11" :> Get '[JSON] Int
308+ :<|> "12" :> Get '[JSON] Int
309+ :<|> "13" :> Get '[JSON] Int
310+ :<|> "14" :> Get '[JSON] Int
311+ :<|> "15" :> Get '[JSON] Int
312+ :<|> "16" :> Get '[JSON] Int
313+
314+
315+type OctetAPI = Get '[OctetStream] BS.ByteString
316+
317+octetAPI :: Proxy OctetAPI
318+octetAPI = Proxy
319+
320+serverOctetAPI :: IO (Server OctetAPI)
321+serverOctetAPI = return $ return "blah"
322+
323+type JsonApi = "jsonComparison" :> Get '[OctetStream] BS.ByteString
324+
325+jsonApi :: Proxy JsonApi
326+jsonApi = Proxy
327+
328+jsonServer1 :: IO (Server JsonApi)
329+jsonServer1 = return $ return "{ \"b\": [\"b\"], \"a\": 1 }" -- whitespace, ordering different
330+
331+jsonServer2 :: IO (Server JsonApi)
332+jsonServer2 = return $ return "{\"a\": 1,\"b\":[\"b\"]}"
333+
334+jsonServer3 :: IO (Server JsonApi)
335+jsonServer3 = return $ return "{\"a\": 2, \"b\": [\"b\"]}"
336+
337+jsonServer4 :: IO (Server JsonApi)
338+jsonServer4 = return $ return "{\"c\": 1, \"d\": [\"b\"]}"
339+
340+
341+ctx :: Context '[BasicAuthCheck ()]
342+ctx = BasicAuthCheck (const . return $ NoSuchUser) :. EmptyContext
343+------------------------------------------------------------------------------
344+-- Utils
345+------------------------------------------------------------------------------
346+evalExample :: (Example e, Arg e ~ ()) => e -> IO EvalResult
347+evalExample e = do
348+ r <- safeEvaluateExample e defaultParams ($ ()) progCallback
349+ case resultStatus r of
350+ Success -> return $ AllGood
351+ Failure _ reason -> return $ FailedWith $ show reason
352+ Pending {} -> error "should not happen"
353+ where
354+ progCallback _ = return ()
355+
356+data EvalResult
357+ = AnException SomeException
358+ | AllGood
359+ | FailedWith String
360+ deriving (Show)
361+
362+
363+args :: Args
364+args = defaultArgs { maxSuccess = noOfTestCases }
365+
366+noOfTestCases :: Int
367+#if LONG_TESTS
368+noOfTestCases = 20000
369+#else
370+noOfTestCases = 1000
371+#endif