git.iko.soy/servant-quickcheck
on master help sign in
.gitignore+5
since splitsnapshots
unifiedsplit
@@ -1 +1,6 @@
1doc/_build/
1doc/_build/
2+scripts/
3+samples/
4+test-servers/
5+/doc/
6+.stack-work/
.travis.yml+154−30
@@ -1,34 +1,158 @@
1−sudo: false
1+# This Travis job script has been generated by a script via
2−
2+#
3+# haskell-ci '--config=cabal.haskell-ci' 'servant-quickcheck.cabal'
4+#
5+# To regenerate the script (for example after adjusting tested-with) run
6+#
7+# haskell-ci regenerate
8+#
9+# For more information, see https://github.com/haskell-CI/haskell-ci
10+#
11+# version: 0.10.1
12+#
13+version: ~> 1.0
3language: c
14language: c
4−
15+os: linux
5−env:
16+dist: xenial
6− - GHCVER=7.8.4
17+git:
7− - GHCVER=7.10.2
18+ # whether to recursively clone submodules
8−
19+ submodules: false
9−addons:
20+branches:
10− apt:
21+ only:
11− sources:
22+ - master
12− - hvr-ghc
23+cache:
13− packages:
24+ directories:
14− - ghc-7.8.4
25+ - $HOME/.cabal/packages
15− - ghc-7.10.2
26+ - $HOME/.cabal/store
16− - cabal-install-1.22
27+ - $HOME/.hlint
17− - libgmp-dev
28+before_cache:
18− - wrk
29+ - rm -fv $CABALHOME/packages/hackage.haskell.org/build-reports.log
19−
30+ # remove files that are regenerated by 'cabal update'
31+ - rm -fv $CABALHOME/packages/hackage.haskell.org/00-index.*
32+ - rm -fv $CABALHOME/packages/hackage.haskell.org/*.json
33+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.cache
34+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar
35+ - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar.idx
36+ - rm -rfv $CABALHOME/packages/head.hackage
37+jobs:
38+ include:
39+ - compiler: ghc-8.8.3
40+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.8.3","cabal-install-3.2"]}}
41+ os: linux
42+ - compiler: ghc-8.6.5
43+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.6.5","cabal-install-3.2"]}}
44+ os: linux
45+ - compiler: ghc-8.4.4
46+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.4.4","cabal-install-3.2"]}}
47+ os: linux
48+ - compiler: ghc-8.2.2
49+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.2.2","cabal-install-3.2"]}}
50+ os: linux
51+ - compiler: ghc-8.0.2
52+ addons: {"apt":{"sources":[{"sourceline":"deb http://ppa.launchpad.net/hvr/ghc/ubuntu xenial main","key_url":"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x063dab2bdc0b3f9fcebc378bff3aeacef6f88286"}],"packages":["ghc-8.0.2","cabal-install-3.2"]}}
53+ os: linux
54+before_install:
55+ - HC=$(echo "/opt/$CC/bin/ghc" | sed 's/-/\//')
56+ - WITHCOMPILER="-w $HC"
57+ - HADDOCK=$(echo "/opt/$CC/bin/haddock" | sed 's/-/\//')
58+ - HCPKG="$HC-pkg"
59+ - unset CC
60+ - CABAL=/opt/ghc/bin/cabal
61+ - CABALHOME=$HOME/.cabal
62+ - export PATH="$CABALHOME/bin:$PATH"
63+ - TOP=$(pwd)
64+ - "HCNUMVER=$(${HC} --numeric-version|perl -ne '/^(\\d+)\\.(\\d+)\\.(\\d+)(\\.(\\d+))?$/; print(10000 * $1 + 100 * $2 + ($3 == 0 ? $5 != 1 : $3))')"
65+ - echo $HCNUMVER
66+ - CABAL="$CABAL -vnormal+nowrap"
67+ - set -o pipefail
68+ - TEST=--enable-tests
69+ - BENCH=--enable-benchmarks
70+ - HEADHACKAGE=false
71+ - rm -f $CABALHOME/config
72+ - |
73+ echo "verbose: normal +nowrap +markoutput" >> $CABALHOME/config
74+ echo "remote-build-reporting: anonymous" >> $CABALHOME/config
75+ echo "write-ghc-environment-files: always" >> $CABALHOME/config
76+ echo "remote-repo-cache: $CABALHOME/packages" >> $CABALHOME/config
77+ echo "logs-dir: $CABALHOME/logs" >> $CABALHOME/config
78+ echo "world-file: $CABALHOME/world" >> $CABALHOME/config
79+ echo "extra-prog-path: $CABALHOME/bin" >> $CABALHOME/config
80+ echo "symlink-bindir: $CABALHOME/bin" >> $CABALHOME/config
81+ echo "installdir: $CABALHOME/bin" >> $CABALHOME/config
82+ echo "build-summary: $CABALHOME/logs/build.log" >> $CABALHOME/config
83+ echo "store-dir: $CABALHOME/store" >> $CABALHOME/config
84+ echo "install-dirs user" >> $CABALHOME/config
85+ echo " prefix: $CABALHOME" >> $CABALHOME/config
86+ echo "repository hackage.haskell.org" >> $CABALHOME/config
87+ echo " url: http://hackage.haskell.org/" >> $CABALHOME/config
20install:
88install:
21− - (mkdir -p $HOME/.local/bin && cd $HOME/.local/bin && wget https://zalora-public.s3.amazonaws.com/tinc && chmod +x tinc)
89+ - ${CABAL} --version
22− - export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/1.22/bin:$PATH
90+ - echo "$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]"
23− - ghc --version
91+ - |
24− - cabal --version
92+ echo "program-default-options" >> $CABALHOME/config
93+ echo " ghc-options: $GHCJOBS +RTS -M6G -RTS" >> $CABALHOME/config
94+ - cat $CABALHOME/config
95+ - rm -fv cabal.project cabal.project.local cabal.project.freeze
25− - travis_retry cabal update
96+ - travis_retry ${CABAL} v2-update -v
26− - sed -i 's/^jobs:/-- jobs:/' ${HOME}/.cabal/config
97+ # Generate cabal.project
27−
98+ - rm -rf cabal.project cabal.project.local cabal.project.freeze
99+ - touch cabal.project
100+ - |
101+ echo "packages: ." >> cabal.project
102+ - if [ $HCNUMVER -ge 80200 ] ; then echo 'package servant-quickcheck' >> cabal.project ; fi
103+ - "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
104+ - |
105+ - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(servant-quickcheck)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
106+ - cat cabal.project || true
107+ - cat cabal.project.local || true
108+ - if [ -f "./configure.ac" ]; then (cd "." && autoreconf -i); fi
109+ - ${CABAL} v2-freeze $WITHCOMPILER ${TEST} ${BENCH}
110+ - "cat cabal.project.freeze | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'"
111+ - rm cabal.project.freeze
112+ - travis_wait 40 ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} --dep -j2 all
113+ - travis_wait 40 ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --dep -j2 all
28script:
114script:
29− - tinc && cabal configure --enable-tests && cabal build && cabal test
115+ - DISTDIR=$(mktemp -d /tmp/dist-test.XXXX)
30− #- (cd doc && tinc cabal configure --enable-tests && cabal build && cabal test)
116+ # Packaging...
117+ - ${CABAL} v2-sdist all
118+ # Unpacking...
119+ - mv dist-newstyle/sdist/*.tar.gz ${DISTDIR}/
120+ - cd ${DISTDIR} || false
121+ - find . -maxdepth 1 -type f -name '*.tar.gz' -exec tar -xvf '{}' \;
122+ - find . -maxdepth 1 -type f -name '*.tar.gz' -exec rm '{}' \;
123+ - PKGDIR_servant_quickcheck="$(find . -maxdepth 1 -type d -regex '.*/servant-quickcheck-[0-9.]*')"
124+ # Generate cabal.project
125+ - rm -rf cabal.project cabal.project.local cabal.project.freeze
126+ - touch cabal.project
127+ - |
128+ echo "packages: ${PKGDIR_servant_quickcheck}" >> cabal.project
129+ - if [ $HCNUMVER -ge 80200 ] ; then echo 'package servant-quickcheck' >> cabal.project ; fi
130+ - "if [ $HCNUMVER -ge 80200 ] ; then echo ' ghc-options: -Werror=missing-methods' >> cabal.project ; fi"
131+ - |
132+ - "for pkg in $($HCPKG list --simple-output); do echo $pkg | sed 's/-[^-]*$//' | (grep -vE -- '^(servant-quickcheck)$' || true) | sed 's/^/constraints: /' | sed 's/$/ installed/' >> cabal.project.local; done"
133+ - cat cabal.project || true
134+ - cat cabal.project.local || true
135+ # Building...
136+ # this builds all libraries and executables (without tests/benchmarks)
137+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all
138+ # Building with tests and benchmarks...
139+ # build & run tests, build benchmarks
140+ - ${CABAL} v2-build $WITHCOMPILER ${TEST} ${BENCH} all
141+ # Testing...
142+ - ${CABAL} v2-test $WITHCOMPILER ${TEST} ${BENCH} all
143+ # cabal check...
144+ - (cd ${PKGDIR_servant_quickcheck} && ${CABAL} -vnormal check)
145+ # haddock...
146+ - ${CABAL} v2-haddock $WITHCOMPILER --with-haddock $HADDOCK ${TEST} ${BENCH} all
147+ # Building without installed constraints for packages in global-db...
148+ - rm -f cabal.project.local
149+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks all
150+ # Constraint sets
151+ - rm -rf cabal.project.local
152+ # Constraint set base-compat-0.10
153+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --constraint='base-compat ==0.10.*' all
154+ # Constraint set base-compat-0.11
155+ - ${CABAL} v2-build $WITHCOMPILER --disable-tests --disable-benchmarks --constraint='base-compat ==0.11.*' all
31
156
32−cache:
157+# REGENDATA ("0.10.1",["--config=cabal.haskell-ci","servant-quickcheck.cabal"])
33− directories:
158+# EOF
34− - $HOME/.tinc/cache
CHANGELOG.yamlnew file+214
@@ -0,0 +1,214 @@
1+releases:
2+ - version: "0.0.9.0"
3+ changes:
4+ - description: "Support servant-0.15 (#65)"
5+ authors: fizruk
6+ date: 2020-06-25
7+
8+ - description: "Relax constraints for GHC 8.10 (#70)"
9+ authors: felixmulder
10+ date: 2020-06-20
11+
12+ - version: "0.0.8.0"
13+ changes:
14+ - description: Support for servant-0.17
15+ authors: phadej
16+ date: 2019-01-23
17+
18+ - version: "0.0.7.3"
19+ changes:
20+ - description: Support for servant-0.14
21+ issue: 53
22+ authors: phadej
23+ date: 2018-06-12
24+
25+ - description: Fix a failure from OnlyJsonObjects when there is no content-type.
26+ issue: 55
27+ authors: Phenitei
28+ date: 2018-08-27
29+
30+ - description: A bug fix where notAllowedContainsAllowHeader would print the initial request alongside the failure instead of the request causing the failure when it failed.
31+ issue: 57
32+ authors: Phenitei
33+ date: 2018-08-29
34+
35+ - description: QuickCheck 2.12 compatibility
36+ issue: 58
37+ authors: parsonsmatt
38+ date: 2018-10-12
39+
40+ - description: GHC 8.6 compatibility
41+ issue: 59
42+ authors: phadej
43+ date: 2018-10-15
44+
45+ - version: "0.0.7.2"
46+ changes:
47+
48+ - description: Allow client to pass an HTTP Manager in to functions
49+ issue: 47
50+ authors: parsonsmatt
51+ date: 2018-05-10
52+
53+ - description: Fix "should not happen" error when exceptions are thrown
54+ issue: 48
55+ authors: parsonsmatt
56+ date: 2018-05-10
57+
58+ - version: "0.0.7.0"
59+ changes:
60+
61+ - description: Support for base-compat-0.10
62+ issue: none
63+ authors: phadej
64+ date: 2018-04-12
65+
66+ - version: "0.0.7.0"
67+ changes:
68+
69+ - description: Support for GHC-8.4.1
70+ issue: none
71+ authors: phadej
72+ date: 2018-03-23
73+
74+ - description: Requires hspec-2.5
75+ issue: none
76+ authors: phadej
77+ date: 2018-03-23
78+
79+ - version: "0.0.6.0"
80+ changes:
81+
82+ - description: Support for servant-0.12
83+ issue: none
84+ authors: phadej
85+ date: 2018-02-09
86+
87+ - version: "0.0.5.0"
88+ changes:
89+
90+ - description: Export forgotten predicates
91+ issue: none
92+ pr: 40
93+ authors: Phenitei
94+ date: 2017-12-14
95+
96+ - version: "0.0.4"
97+ changes:
98+
99+ - description: Support for Servant 0.12
100+ issue: none
101+ authors: phadej
102+ date: 2017-11-07
103+
104+ - description: Support for Servant 0.11
105+ issue: none
106+ pr: 32
107+ authors: adinapoli-iohk
108+ date: 2017-10-18
109+ notes: Includes 0-weighted instance for EmptyAPI
110+
111+ - version: "0.0.3.0"
112+ changes:
113+
114+ - description: Add jsonEquality
115+ issue: 2
116+ pr: 30
117+ authors: erewok
118+ date: 2017-10-15
119+
120+ - description: Support hspec >= 2.4
121+ issue: 27
122+ pr: 29
123+ authors: erewok, jkarni
124+ date: 2017-10-15
125+
126+ - version: "0.0.2.4"
127+ changes:
128+
129+ - description: Don't append slashes to paths
130+ issue: 22
131+ authors: declension
132+ date: 2017-03-11
133+
134+ - version: "0.0.2.3"
135+ changes:
136+
137+ - description: Fix QueryParam and QueryFlag requests
138+ issue: 23
139+ authors: declension
140+ date: 2017-03-10
141+
142+ - version: "0.0.2.2"
143+ changes:
144+
145+ - description: Make onlyJsonObjects succeed in non-JSON endpoints
146+ issue: 20
147+ authors: jkarni
148+ date: 2016-10-18
149+
150+
151+ - version: "0.0.2.1"
152+ changes:
153+
154+ - description: Add notLongerThan predicate
155+ pr: 17
156+ authors: jkarni
157+ date: 2016-10-05
158+
159+ - description: Add getsHaveLastModifiedHeader predicate
160+ pr: none
161+ authors: jkarni
162+ date: 2016-10-03
163+
164+ - description: Raise upper bounds
165+ notes: >
166+ For Quickcheck, aeson, http-client, servant, servant-client and
167+ servant-server.
168+ pr: none
169+ authors: jkarni
170+ date: 2016-10-03
171+
172+
173+ - version: "0.0.2.0"
174+ changes:
175+
176+ - description: Update CHANGELOG to YAML syntax.
177+ pr: 16
178+ authors: jkarni
179+ date: 2016-09-14
180+
181+ - description: Support new CaptureAll combinator
182+ pr: 16
183+ authors: jkarni
184+ date: 2016-09-14
185+
186+ - description: Support GHC 8
187+ pr: 16
188+ authors: jkarni
189+ date: 2016-09-14
190+
191+ - version: "0.0.1.1"
192+ changes:
193+
194+ - description: Exclude GHC 7.8 (by bumping base lower bound to 4.8)
195+ pr: none
196+ authors: jkarni
197+
198+ - description: More generous bounds for other packages.
199+ pr: none
200+ authors: jkarni
201+
202+ - version: "0.0.1.0"
203+ changes:
204+
205+ - description: Better error messages.
206+ notes: >
207+ Error messages now contain failing predicate, failing response and
208+ (except for response predicates), failing requests.
209+ pr: none
210+ authors: jkarni
211+
212+ - description: Signicant changes to RequestPredicate and ResponsePredicate types.
213+ pr: none
214+ authors: jkarni
README.mdnew file+25
@@ -0,0 +1,25 @@
1+# servant-quickcheck
2+
3+`servant-quickcheck` provides tools to test properties across entire APIs.
4+Rather than writing the same tests for each endpoint, with `servant-quickcheck`
5+you can simply specify properties every endpoint must meet once and for all.
6+For example:
7+
8+``` haskell
9+followsBestPractices :: Spec
10+followsBestPractices = describe "my API" $ do
11+
12+ it "follows best practices" $ do
13+ withServantServer myAPI myServer $ \burl ->
14+ serverSatisfies api burl stdArgs
15+ ( not500
16+ <%> onlyJsonObjects
17+ <%> getsHaveCacheControlHeader
18+ <%> headsHaveCacheControlHeader
19+ <%> mempty)
20+```
21+
22+Additionally, `servant-quickcheck` provides a `serversEqual` function that
23+generates arbitrary requests (that conform to the description of an API) and
24+tests that two servers respond identically to them. This can be useful when
25+refactoring or rewriting an API that should not change.
cabal.haskell-cinew file+7
@@ -0,0 +1,7 @@
1+branches: master
2+
3+constraint-set base-compat-0.10
4+ constraints: base-compat ==0.10.*
5+
6+constraint-set base-compat-0.11
7+ constraints: base-compat ==0.11.*
cabal.projectnew file+4
@@ -0,0 +1,4 @@
1+packages: .
2+tests: true
3+
4+allow-newer: servant-blaze:servant
doc/doc.cabal+36−18
@@ -7,24 +7,39 @@ maintainer: jkarni@gmail.com
7build-type: Simple
7build-type: Simple
8cabal-version: >=1.10
8cabal-version: >=1.10
9
9
10−library
10+-- library
11− exposed-modules: ServersEqual
11+-- exposed-modules: ServersEqual
12− other-extensions: DataKinds, TypeOperators
12+-- other-extensions: DataKinds, TypeOperators
13− build-depends: base >=4.8 && <4.9
13+-- build-depends: base >=4.8 && <4.9
14− , servant-server == 0.7.*
14+-- , servant-server == 0.7.*
15− , servant-quickcheck
15+-- , servant-quickcheck
16− , servant-client
16+-- , servant-client
17− , QuickCheck
17+-- , QuickCheck
18− , stm
18+-- , stm
19− , containers
19+-- , containers
20− , transformers
20+-- , transformers
21− , warp
21+-- , warp
22− , aeson
22+-- , aeson
23− ghc-options: -Wall -Werror -pgmL markdown-unlit
23+-- ghc-options: -Wall -Werror -pgmL markdown-unlit
24− default-language: Haskell2010
24+-- default-language: Haskell2010
25+--
26+-- executable doc
27+-- main-is: Main.hs
28+-- build-depends: base >=4.8 && <4.9
29+-- , servant-server == 0.7.*
30+-- , servant-quickcheck
31+-- , servant-client
32+-- , QuickCheck
33+-- , stm
34+-- , containers
35+-- , transformers
36+-- , warp
37+-- , aeson
38+-- default-language: Haskell2010
39+-- ghc-options: -Wall -Werror -pgmL markdown-unlit
25
40
26−executable doc
41+executable announcement
27− main-is: Main.hs
42+ main-is: Announcement.lhs
28 build-depends: base >=4.8 && <4.9
43 build-depends: base >=4.8 && <4.9
29 , servant-server == 0.7.*
44 , servant-server == 0.7.*
30 , servant-quickcheck
45 , servant-quickcheck
@@ -35,5 +50,8 @@ executable doc
35 , transformers
50 , transformers
36 , warp
51 , warp
37 , aeson
52 , aeson
53+ , hspec
54+ , postgresql-simple
55+ , text
38 default-language: Haskell2010
56 default-language: Haskell2010
39− ghc-options: -Wall -Werror -pgmL markdown-unlit
57+ ghc-options: -Wall
doc/posts/Announcement.anansinew file+330
@@ -0,0 +1,330 @@
1+:loom anansi.markdown
2+
3+# Announcing servant-quickcheck
4+
5+Some time ago, we released `servant-mock`. The idea behind it is to use
6+`QuickCheck` to create a mock server that accords with a servant API. Not long
7+after, we started thinking about an analog that would, instead of mocking a
8+server, mock a client instead - i.e., generate random requests that conform to
9+an API description.
10+
11+This is much closer to the traditional use of `QuickCheck`. The most obvious
12+use-case is checking that properties hold of an *entire* server rather than of
13+individual endpoints. (But there are other uses that you can skip to if they
14+sound more interesting.)
15+
16+## `serverSatisfies`
17+
18+A useful guideline when writing and maintaing software is that, if there isn't
19+a test for a behaviour or property, sooner or later that property will be broken.
20+Another important perspective is that tests are a form of documentation - the
21+present developer telling future ones "this matters, and should be this way".
22+
23+The advantage of using tests for this form of documentation is that there's
24+simply too much information to convey, some of it only relevant to very specific
25+use cases, and rather than overload developers with an inexhaustible quantity of
26+details that would be hard to keep track of or remember, tests are a mechanism
27+of reminding developers of *only the relevant information, at the right time*.
28+<<EXAMPLE>>.
29+
30+We might hope that we could use tests to communicate the wide array of best
31+practices that have developed around APIs. About to return a top-level integer
32+in JSON? A test should say that's bad practice. About to not catch exceptions
33+and give a more meaningful HTTP status code? Another test there to stop you.
34+
35+Traditionally, in web services these things get done at the level of *individual*
36+endpoints. But this means that if a developer who hasn't had extensive experience with web
37+programming best practices writes a *new* endpoint which *does* return a top-level
38+integer literal, there's no test there to stop her. Code review might help, but
39+code review is much more error prone than tests, and really only meant for those
40+things that are too subtle to automate. (Indeed, if code review were such a reliable
41+defense mechanism against bugs and bad code, why have tests and linters at all?)
42+
43+The problem, then, with thinking about tests as only existing at the level of individual
44+endpoints is that there are no tests *for* tests - tests that check that new
45+behaviour and tests conforms to higher-level, more general best practices.
46+
47+`servant-quickcheck` aims to solve that. It allows describing properties that
48+*all* endpoints must satisfy. If a new endpoint comes along, it too will be
49+tested for that property, without any further work.
50+
51+Why isn't this idea already popular? Well, most web frameworks don't have a
52+reified description of APIs (beyond perhaps the routes). When you don't know
53+what the endpoints of an application are, and what request body they expect,
54+trying to generate arbitrary requests is almost entirely going to result in
55+404s (not found) and 400s (bad request). Maybe one in a thousand requests will
56+actually test a handler. Not very useful.
57+
58+`servant` applications, on the other hand, have a machine-readable API description
59+already available. And they already associate "correct" requests with particular
60+types. It's a small step, therefore, to generate 'arbitrary' values for these
61+requests, and all of them will go through to your handlers. (Note: all of the
62+uses of `servant-quickcheck` work with applications *not* written with servant-server -
63+and indeed not *in Haskell - but the API must be described with the servant
64+DSL.)
65+
66+Let's see how this works in practice. As a running example, let's use a simple
67+service that allows adding, removing, and querying biological species. Our SQL
68+schema is:
69+
70+:d schema.sql
71+
72+CREATE TABLE genus (
73+ genus_name text PRIMARY KEY,
74+ genus_family text NOT NULL
75+);
76+
77+CREATE TABLE species (
78+ species_name text PRIMARY KEY,
79+ species_genus text NOT NULL REFERENCES genus (genus_name)
80+)
81+:
82+
83+And our actual application:
84+
85+:d Main.hs
86+{-# LANGUAGE DataKinds #-}
87+{-# LANGUAGE DeriveAnyClass #-}
88+{-# LANGUAGE DeriveGeneric #-}
89+{-# LANGUAGE TypeOperators #-}
90+{-# LANGUAGE OverloadedStrings #-}
91+{-# LANGUAGE RecordWildCards #-}
92+module Main where
93+
94+import Servant
95+import Data.Aeson
96+import Database.PostgreSQL.Simple
97+import GHC.Generics (Generic)
98+import Data.Text (Text)
99+import Network.Wai.Handler.Warp
100+import Control.Monad.IO.Class (liftIO)
101+
102+type API
103+ = "species" :> (Capture "species-name" Text :> ( Get '[JSON] Species
104+ :<|> Delete '[JSON] ())
105+ :<|> ReqBody '[JSON] Species :> Post '[JSON] ()
106+ :<|> Get '[JSON] [Species])
107+
108+api :: Proxy API
109+api = Proxy
110+
111+data Species = Species
112+ { speciesName :: Text
113+ , speciesGenus :: Text
114+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
115+
116+data Genus = Genus
117+ { genusName :: Text
118+ , genusFamily :: Text
119+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
120+
121+instance FromRow Genus
122+instance FromRow Species
123+
124+server :: Connection -> Server API
125+server conn = ((\sname -> liftIO (lookupSpecies conn sname)
126+ :<|> liftIO (deleteSpecies conn sname))
127+ :<|> (\species -> liftIO $ insertSpecies conn species))
128+ :<|> (liftIO $ allSpecies conn)
129+
130+lookupSpecies :: Connection -> Text -> IO Species
131+lookupSpecies conn name = do
132+ [s] <- query conn "SELECT * FROM species WHERE species_name = ?" (Only name)
133+ return s
134+
135+deleteSpecies :: Connection -> Text -> IO ()
136+deleteSpecies conn name = do
137+ _ <- execute conn "DELETE FROM species WHERE species_name = ?" (Only name)
138+ return ()
139+
140+insertSpecies :: Connection -> Species -> IO ()
141+insertSpecies conn Species{..} = do
142+ _ <- execute conn "INSERT INTO species (species_name, species_genus) VALUES (?)" (speciesName, speciesGenus)
143+ return ()
144+
145+allSpecies :: Connection -> IO [Species]
146+allSpecies conn = do
147+ query_ conn "SELECT * FROM species"
148+
149+main :: IO ()
150+main = do
151+ conn <- connectPostgreSQL "dbname=servant-quickcheck"
152+ run 8090 (serve api $ server conn)
153+:
154+
155+(You'll also also need to run:
156+
157+```
158+createdb servant-quickcheck
159+psql --file schema.sql -d servant-quickcheck
160+```
161+
162+If you want to run this example.)
163+
164+This is a plausible effort. You might want to spend a moment thinking about what
165+could be improved.
166+
167+Here are some `servant-quickcheck`-based tests for this API:
168+
169+:d Spec.hs
170+
171+{-# LANGUAGE OverloadedStrings #-}
172+module Spec (main) where
173+
174+import Main (server, api, Species(..))
175+import Test.Hspec
176+import Test.QuickCheck.Instances
177+import Servant.QuickCheck
178+import Test.QuickCheck (Arbitrary(..))
179+import Database.PostgreSQL.Simple (connectPostgreSQL)
180+
181+spec :: Spec
182+spec = describe "the species application" $ beforeAll check $ do
183+ let pserver = do
184+ conn <- connectPostgreSQL "dbname=servant-quickcheck"
185+ return $ server conn
186+
187+
188+ it "should not return 500s" $ do
189+ withServantServer api pserver $ \url ->
190+ serverSatisfies api url defaultArgs (not500 <%> mempty)
191+
192+ it "should not return top-level json" $ do
193+ withServantServer api pserver $ \url ->
194+ serverSatisfies api url defaultArgs (onlyJsonObjects <%> mempty)
195+
196+ it "should return valid locations for 201" $ do
197+ withServantServer api pserver $ \url ->
198+ serverSatisfies api url defaultArgs (createContainsValidLocation <%> mempty)
199+
200+ it "honours Accept header" $ do
201+ withServantServer api pserver $ \url ->
202+ serverSatisfies api url defaultArgs (honoursAcceptHeader <%> mempty)
203+
204+ it "405s contain Allow header" $ do
205+ withServantServer api pserver $ \url ->
206+ serverSatisfies api url defaultArgs (notAllowedContainsValidAllow <%> mempty)
207+
208+ it "should contain WWW-Authenticate header when returning 401s" $ do
209+ withServantServer api pserver $ \url ->
210+ serverSatisfies api url defaultArgs (unauthorizedContainsWWWAuthenticate <%> mempty)
211+
212+ it "GETs should have Cache-Control header" $ do
213+ withServantServer api pserver $ \url ->
214+ serverSatisfies api url defaultArgs (getsHaveCacheControlHeader <%> mempty)
215+
216+ it "HEADs should have Cache-Control header" $ do
217+ withServantServer api pserver $ \url ->
218+ serverSatisfies api url defaultArgs (headsHaveCacheControlHeader <%> mempty)
219+
220+main :: IO ()
221+main = do
222+ hspec spec
223+
224+instance Arbitrary Species where
225+ arbitrary = Species <$> arbitrary <*> arbitrary
226+:
227+
228+
229+I won't go through all of the failures; as this is a literate haskell file, you
230+can go through them yourself if you're interested. But worth mentioning:
231+
232+ - The possible pattern match failure in `lookupSpecies` is caught.
233+ - Returning a top-level list is caught.
234+ - Not having a link for the `PostCreated` is caught.
235+
236+This last test failure illustrates an important point.
237+
238+
239+This was an example created with the knowledge of what it was supposed to
240+exemplify. To try to get a more accurate assessment of the practical usefulness
241+of `servant-quickcheck`, I tried running `serverSatisfies` with a few
242+predicates over some of the open-source `servant` servers I could find, and
243+results were also promising.
244+
245+There are probably a lot of other interesting properties that one might to add
246+besides those I've included. As an example, we could have a property that
247+all HTML is checked against, which is sometimes tricky for HTML that's
248+generated dynamically. Or check that every page has a Portuguese translation.
249+
250+### Why best practices are good
251+
252+As a side note: you might have wondered "why bother with API best practices?".
253+It is, it has to be said, a lot of extra (as in not only getting the feature done)
254+work to do, for dubious benefit. And indeed, the relevance of discoverability, for
255+example, unclear, since not that many tools use it as perhaps was anticipated.
256+
257+But `servant-quickcheck` both makes it *easier* to conform to best practices,
258+and exemplifies their advantage in enabling better tooling. If we pick 201 (Success, the 'resource' was
259+created), rather than the more generic 200 (Success), and do a *little* more work
260+by knowing to make this decision, `servant-quickcheck` knows this means there
261+should be some representation of the resource created. So it knows to ask you
262+for a link to it (the RFC creators thought to ask for this). And if you do (again,
263+a little more work), `servant-quickcheck` will know to try to look at that
264+resource by following the link, checking that it's not broken, and maybe even
265+returns a response that equivalent to the original POST request). And then it
266+finds a real bug - your application allows species with '/' in their name to
267+be created, but not queried with a 'GET' for! This, I think, is already a win.
268+
269+
270+## `serversEqual`
271+
272+There's another very appealing application of the ability to generate "sensible"
273+arbitrary requests. It's for testing that two applications are equal. We can generate arbitrary
274+requests, send them to both servers (in the same order), and check that the responses
275+are equivalent. (This was, incidentally, one of the first applications of
276+`servant-client`, albeit in a much more manual way, when we rewrote a microservice
277+originally in Python in Haskell.) Generally with rewrites, even if there's some
278+behaviour that isn't optimal, perhaps a lot of things already depend on that service
279+and make interace poorly with "improvements", so it makes sense to first mimick
280+*exactly* the original behaviour, and only then aim for improvements.
281+
282+`servant-quickcheck` provides a single function, `serversEqual`, that attempts
283+to verify the equivalence of servers. Since some aspects of responses might not
284+be relevant (for example, whether the the `Server` header is the same, or whether
285+two JSON responses have the same formatting), it allows you to provide a custom
286+equivalence function. Other than that, you need only provide an API type and two
287+URLs for testing, and the rest `serversEqual` handles.
288+
289+## Future directions: benchmarking
290+
291+What else could benefit from tooling that can automatically generate sensible
292+(*vis-a-vis* a particular application's expectations) requests?
293+
294+One area is extensive automatic benchmarking. Currently we use tools such as
295+`ab`, `wrk`, `httperf` in a very manual way - we pick a particular request that
296+we are interested in, and write a request that gets made thousands of times.
297+But now we can have a multiplicity of requests to benchmark with! This allows
298+*finding* slow endpoints, as well as (I would imagine, though I haven't actually
299+tried this yet) finding synchronization issues that make threads wait for too
300+long (such as waiting on an MVar that's not really needed), bad asymptotics
301+with respect to some other type of request.
302+
303+(On this last point, imagine not having an index in a database for "people",
304+ and having a tool that discovers that the latency on a search by first name
305+ grows linearly with the number of POST requests to a *different* endpoint! We'd
306+ need to do some work to do this well, possibly involving some machine
307+ learning, but it's an interesting and probably useful idea.)
308+
309+
310+# Conclusion
311+
312+I hope this library presents some useful functionality already, but I hope
313+you'll also think how it could be improved!
314+
315+There'll be a few more packages in the comings weeks - check back soon!
316+
317+**Note**: This post is an anansi literate file that generates multiple source
318+files. They are:
319+
320+:f Main.hs
321+|Main.hs|
322+:
323+
324+:f schema.sql
325+|schema.sql|
326+:
327+
328+:f Spec.hs
329+|Spec.hs|
330+:
doc/posts/LICENSEnew file+30
@@ -0,0 +1,30 @@
1+Copyright (c) 2016, Julian K. Arni
2+
3+All rights reserved.
4+
5+Redistribution and use in source and binary forms, with or without
6+modification, are permitted provided that the following conditions are met:
7+
8+ * Redistributions of source code must retain the above copyright
9+ notice, this list of conditions and the following disclaimer.
10+
11+ * Redistributions in binary form must reproduce the above
12+ copyright notice, this list of conditions and the following
13+ disclaimer in the documentation and/or other materials provided
14+ with the distribution.
15+
16+ * Neither the name of Julian K. Arni nor the names of other
17+ contributors may be used to endorse or promote products derived
18+ from this software without specific prior written permission.
19+
20+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
doc/posts/Makefilenew file+25
@@ -0,0 +1,25 @@
1+FILES = src/Main.hs src/Spec.hs src/schema.sql
2+
3+src/$(FILES): Announcement.anansi
4+ anansi tangle -o "src" Announcement.anansi
5+
6+announcement.md: Announcement.anansi
7+ anansi weave -o "announcement.tmp" Announcement.anansi
8+ cat announcement.tmp | tr -d '«' | tr -d '»' > announcement.md
9+ rm announcement.tmp
10+
11+.stack-work/bin/posts: $(FILES) stack.yaml posts.cabal
12+ stack build
13+
14+announcement.html: announcement.md
15+ pandoc announcement.md -t html > announcement.html
16+
17+run: .stack-work/bin/posts
18+ stack exec posts
19+
20+test: .stack-work/bin/posts
21+ stack test
22+
23+post: announcement.html
24+
25+.PHONY: post run test
doc/posts/Setup.hsnew file+2
@@ -0,0 +1,2 @@
1+import Distribution.Simple
2+main = defaultMain
doc/posts/announcement.mdnew file+318
@@ -0,0 +1,318 @@
1+
2+# Announcing servant-quickcheck
3+
4+Some time ago, we released `servant-mock`. The idea behind it is to use
5+`QuickCheck` to create a mock server that accords with a servant API. Not long
6+after, we started thinking about an analog that would, instead of mocking a
7+server, mock a client instead - i.e., generate random requests that conform to
8+an API description.
9+
10+This is much closer to the traditional use of `QuickCheck`. The most obvious
11+use-case is checking that properties hold of an *entire* server rather than of
12+individual endpoints. (But there are other uses that you can skip to if they
13+sound more interesting.)
14+
15+## `serverSatisfies`
16+
17+A useful guideline when writing and maintaing software is that, if there isn't
18+a test for a behaviour or property, sooner or later that property will be broken.
19+Another important perspective is that tests are a form of documentation - the
20+present developer telling future ones "this matters, and should be this way".
21+
22+The advantage of using tests for this form of documentation is that there's
23+simply too much information to convey, some of it only relevant to very specific
24+use cases, and rather than overload developers with an inexhaustible quantity of
25+details that would be hard to keep track of or remember, tests are a mechanism
26+of reminding developers of *only the relevant information, at the right time*.
27+<<EXAMPLE>>.
28+
29+We might hope that we could use tests to communicate the wide array of best
30+practices that have developed around APIs. About to return a top-level integer
31+in JSON? A test should say that's bad practice. About to not catch exceptions
32+and give a more meaningful HTTP status code? Another test there to stop you.
33+
34+Traditionally, in web services these things get done at the level of *individual*
35+endpoints. But this means that if a developer who hasn't had extensive experience with web
36+programming best practices writes a *new* endpoint which *does* return a top-level
37+integer literal, there's no test there to stop her. Code review might help, but
38+code review is much more error prone than tests, and really only meant for those
39+things that are too subtle to automate. (Indeed, if code review were such a reliable
40+defense mechanism against bugs and bad code, why have tests and linters at all?)
41+
42+The problem, then, with thinking about tests as only existing at the level of individual
43+endpoints is that there are no tests *for* tests - tests that check that new
44+behaviour and tests conforms to higher-level, more general best practices.
45+
46+`servant-quickcheck` aims to solve that. It allows describing properties that
47+*all* endpoints must satisfy. If a new endpoint comes along, it too will be
48+tested for that property, without any further work.
49+
50+Why isn't this idea already popular? Well, most web frameworks don't have a
51+reified description of APIs (beyond perhaps the routes). When you don't know
52+what the endpoints of an application are, and what request body they expect,
53+trying to generate arbitrary requests is almost entirely going to result in
54+404s (not found) and 400s (bad request). Maybe one in a thousand requests will
55+actually test a handler. Not very useful.
56+
57+`servant` applications, on the other hand, have a machine-readable API description
58+already available. And they already associate "correct" requests with particular
59+types. It's a small step, therefore, to generate 'arbitrary' values for these
60+requests, and all of them will go through to your handlers. (Note: all of the
61+uses of `servant-quickcheck` work with applications *not* written with servant-server -
62+and indeed not *in Haskell - but the API must be described with the servant
63+DSL.)
64+
65+Let's see how this works in practice. As a running example, let's use a simple
66+service that allows adding, removing, and querying biological species. Our SQL
67+schema is:
68+
69+
70+> **schema.sql**
71+
72+>
73+> CREATE TABLE genus (
74+> genus_name text PRIMARY KEY,
75+> genus_family text NOT NULL
76+> );
77+>
78+> CREATE TABLE species (
79+> species_name text PRIMARY KEY,
80+> species_genus text NOT NULL REFERENCES genus (genus_name)
81+> )
82+
83+
84+And our actual application:
85+
86+
87+> **Main.hs**
88+
89+> {-# LANGUAGE DataKinds #-}
90+> {-# LANGUAGE DeriveAnyClass #-}
91+> {-# LANGUAGE DeriveGeneric #-}
92+> {-# LANGUAGE TypeOperators #-}
93+> {-# LANGUAGE OverloadedStrings #-}
94+> {-# LANGUAGE RecordWildCards #-}
95+> module Main where
96+>
97+> import Servant
98+> import Data.Aeson
99+> import Database.PostgreSQL.Simple
100+> import GHC.Generics (Generic)
101+> import Data.Text (Text)
102+> import Network.Wai.Handler.Warp
103+> import Control.Monad.IO.Class (liftIO)
104+>
105+> type API
106+> = "species" :> (Capture "species-name" Text :> ( Get '[JSON] Species
107+> :<|> Delete '[JSON] ())
108+> :<|> ReqBody '[JSON] Species :> Post '[JSON] ())
109+> -- The plural of 'species' is unfortunately also 'species'
110+> :<|> "speciess" :> Get '[JSON] [Species]
111+>
112+> api :: Proxy API
113+> api = Proxy
114+>
115+> data Species = Species
116+> { speciesName :: Text
117+> , speciesGenus :: Text
118+> } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
119+>
120+> data Genus = Genus
121+> { genusName :: Text
122+> , genusFamily :: Text
123+> } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
124+>
125+> instance FromRow Genus
126+> instance FromRow Species
127+>
128+> server :: Connection -> Server API
129+> server conn = ((\sname -> liftIO (lookupSpecies conn sname)
130+> :<|> liftIO (deleteSpecies conn sname))
131+> :<|> (\species -> liftIO $ insertSpecies conn species))
132+> :<|> (liftIO $ allSpecies conn)
133+>
134+> lookupSpecies :: Connection -> Text -> IO Species
135+> lookupSpecies conn name = do
136+> [s] <- query conn "SELECT * FROM species WHERE species_name = ?" (Only name)
137+> return s
138+>
139+> deleteSpecies :: Connection -> Text -> IO ()
140+> deleteSpecies conn name = do
141+> _ <- execute conn "DELETE FROM species WHERE species_name = ?" (Only name)
142+> return ()
143+>
144+> insertSpecies :: Connection -> Species -> IO ()
145+> insertSpecies conn Species{..} = do
146+> _ <- execute conn "INSERT INTO species (species_name, species_genus) VALUES (?)" (speciesName, speciesGenus)
147+> return ()
148+>
149+> allSpecies :: Connection -> IO [Species]
150+> allSpecies conn = do
151+> query_ conn "SELECT * FROM species"
152+>
153+> main :: IO ()
154+> main = do
155+> conn <- connectPostgreSQL "dbname=servant-quickcheck"
156+> run 8090 (serve api $ server conn)
157+
158+
159+(You'll also also need to run:
160+
161+```
162+createdb servant-quickcheck
163+psql --file schema.sql -d servant-quickcheck
164+```
165+
166+If you want to run this example.)
167+
168+This is a plausible effort. You might want to spend a moment thinking about what
169+could be improved.
170+
171+
172+> **Spec.hs**
173+
174+>
175+> {-# LANGUAGE OverloadedStrings #-}
176+> module Spec (main) where
177+>
178+> import Main (server, api, Species(..))
179+> import Test.Hspec
180+> import Test.QuickCheck.Instances
181+> import Servant.QuickCheck
182+> import Test.QuickCheck (Arbitrary(..))
183+> import Database.PostgreSQL.Simple (connectPostgreSQL)
184+>
185+> spec :: Spec
186+> spec = describe "the species application" $ beforeAll check $ do
187+> let pserver = do
188+> conn <- connectPostgreSQL "dbname=servant-quickcheck"
189+> return $ server conn
190+>
191+>
192+> it "should not return 500s" $ do
193+>
194+> it "should not return 500s" $ do
195+> withServantServer api pserver $ \url ->
196+> serverSatisfies api url defaultArgs (not500 <%> mempty)
197+>
198+> it "should not return top-level json" $ do
199+> withServantServer api pserver $ \url ->
200+> serverSatisfies api url defaultArgs (onlyJsonObjects <%> mempty)
201+>
202+> where
203+> check = do
204+> mvar <- newMVar []
205+> withServantServer api pserver $ \url ->
206+> serverSatisfies api url defaultArgs (onlyJsonObjects <%> mempty)
207+>
208+> main :: IO ()
209+> main = do
210+> hspec spec
211+>
212+> instance Arbitrary Species where
213+> arbitrary = Species <$> arbitrary <*> arbitrary
214+
215+
216+But this fails in quite a few ways.
217+
218+
219+<<TODO>>
220+
221+This was an example created with the knowledge of what it was supposed to
222+exemplify. To try to get a more accurate assessment of the practical usefulness
223+of `servant-quickcheck`, I tried running `serverSatisfies` with a few
224+predicates over some of the open-source `servant` servers I could find, and
225+results were also promising.
226+
227+There are probably a lot of other interesting properties that one might to add
228+besides those I've included. As an example, we could have a property that
229+all HTML is checked against, which is sometimes tricky for HTML that's
230+generated dynamically. Or check that every page has a Portuguese translation.
231+
232+### Why best practices are good
233+
234+As a side note: you might have wondered "why bother with API best practices?".
235+It is, it has to be said, a lot of extra (as in not only getting the feature done)
236+work to do, for dubious benefit. And indeed, the relevance of discoverability, for
237+example, unclear, since not that many tools use it as perhaps was anticipated.
238+
239+But `servant-quickcheck` both makes it *easier* to conform to best practices,
240+and exemplifies their advantage in enabling better tooling. If we pick 201 (Success, the 'resource' was
241+created), rather than the more generic 200 (Success), and do a *little* more work
242+by knowing to make this decision, `servant-quickcheck` knows this means there
243+should be some representation of the resource created. So it knows to ask you
244+for a link to it (the RFC creators thought to ask for this). And if you do (again,
245+a little more work), `servant-quickcheck` will know to try to look at that
246+resource by following the link, checking that it's not broken, and maybe even
247+returns a response that equivalent to the original POST request). And then it
248+finds a real bug - your application allows species with '/' in their name to
249+be created, but not queried with a 'GET' for! This, I think, is already a win.
250+
251+
252+## `serversEqual`
253+
254+There's another very appealing application of the ability to generate "sensible"
255+arbitrary requests. It's for testing that two applications are equal. We can generate arbitrary
256+requests, send them to both servers (in the same order), and check that the responses
257+are equivalent. (This was, incidentally, one of the first applications of
258+`servant-client`, albeit in a much more manual way, when we rewrote a microservice
259+originally in Python in Haskell.) Generally with rewrites, even if there's some
260+behaviour that isn't optimal, perhaps a lot of things already depend on that service
261+and make interace poorly with "improvements", so it makes sense to first mimick
262+*exactly* the original behaviour, and only then aim for improvements.
263+
264+`servant-quickcheck` provides a single function, `serversEqual`, that attempts
265+to verify the equivalence of servers. Since some aspects of responses might not
266+be relevant (for example, whether the the `Server` header is the same, or whether
267+two JSON responses have the same formatting), it allows you to provide a custom
268+equivalence function. Other than that, you need only provide an API type and two
269+URLs for testing, and the rest `serversEqual` handles.
270+
271+## Future directions: benchmarking
272+
273+What else could benefit from tooling that can automatically generate sensible
274+(*vis-a-vis* a particular application's expectations) requests?
275+
276+One area is extensive automatic benchmarking. Currently we use tools such as
277+`ab`, `wrk`, `httperf` in a very manual way - we pick a particular request that
278+we are interested in, and write a request that gets made thousands of times.
279+But now we can have a multiplicity of requests to benchmark with! This allows
280+*finding* slow endpoints, as well as (I would imagine, though I haven't actually
281+tried this yet) finding synchronization issues that make threads wait for too
282+long (such as waiting on an MVar that's not really needed), bad asymptotics
283+with respect to some other type of request.
284+
285+(On this last point, imagine not having an index in a database for "people",
286+ and having a tool that discovers that the latency on a search by first name
287+ grows linearly with the number of POST requests to a *different* endpoint! We'd
288+ need to do some work to do this well, possibly involving some machine
289+ learning, but it's an interesting and probably useful idea.)
290+
291+
292+# Conclusion
293+
294+I hope this library presents some useful functionality already, but I hope
295+you'll also think how it could be improved!
296+
297+There'll be a few more packages in the comings weeks - check back soon!
298+
299+**Note**: This post is an anansi literate file that generates multiple source
300+files. They are:
301+
302+
303+> ** Main.hs**
304+
305+> Main.hs
306+
307+
308+
309+> ** schema.sql**
310+
311+> schema.sql
312+
313+
314+
315+> ** Spec.hs**
316+
317+> Spec.hs
318+
doc/posts/posts.cabalnew file+47
@@ -0,0 +1,47 @@
1+-- Initial posts.cabal generated by cabal init. For further documentation,
2+-- see http://haskell.org/cabal/users-guide/
3+
4+name: posts
5+version: 0.1.0.0
6+-- synopsis:
7+-- description:
8+license: BSD3
9+license-file: LICENSE
10+author: Julian K. Arni
11+maintainer: jkarni@gmail.com
12+-- copyright:
13+-- category:
14+build-type: Simple
15+-- extra-source-files:
16+cabal-version: >=1.10
17+
18+executable posts
19+ main-is: Main.hs
20+ build-depends: base >=4.8 && <4.9
21+ , servant-server >=0.5 && <0.8
22+ , aeson >=0.9 && <0.12
23+ , text == 1.*
24+ , warp >=3.0 && <3.3
25+ , transformers >=0.4 && <0.5
26+ , postgresql-simple
27+ hs-source-dirs: src
28+ ghc-options: -Wall -O2 -threaded
29+ default-language: Haskell2010
30+
31+test-suite spec
32+ type: exitcode-stdio-1.0
33+ ghc-options: -Wall -O2 -threaded
34+ default-language: Haskell2010
35+ hs-source-dirs: src
36+ main-is: Spec.hs
37+ build-depends: base == 4.*
38+ , servant-quickcheck
39+ , hspec == 2.*
40+ , servant-server >=0.5 && <0.8
41+ , aeson >=0.9 && <0.12
42+ , text == 1.*
43+ , warp >=3.0 && <3.3
44+ , transformers >=0.4 && <0.5
45+ , postgresql-simple
46+ , quickcheck-instances
47+ , QuickCheck
doc/posts/src/Main.hsnew file+71
@@ -0,0 +1,71 @@
1+#line 296 "Announcement.anansi"
2+
3+#line 86 "Announcement.anansi"
4+{-# LANGUAGE DataKinds #-}
5+{-# LANGUAGE DeriveAnyClass #-}
6+{-# LANGUAGE DeriveGeneric #-}
7+{-# LANGUAGE TypeOperators #-}
8+{-# LANGUAGE OverloadedStrings #-}
9+{-# LANGUAGE RecordWildCards #-}
10+module Main where
11+
12+import Servant
13+import Data.Aeson
14+import Database.PostgreSQL.Simple
15+import GHC.Generics (Generic)
16+import Data.Text (Text)
17+import Network.Wai.Handler.Warp
18+import Control.Monad.IO.Class (liftIO)
19+
20+type API
21+ = "species" :> (Capture "species-name" Text :> ( Get '[JSON] Species
22+ :<|> Delete '[JSON] ())
23+ :<|> ReqBody '[JSON] Species :> Post '[JSON] ())
24+ -- The plural of 'species' is unfortunately also 'species'
25+ :<|> "speciess" :> Get '[JSON] [Species]
26+
27+api :: Proxy API
28+api = Proxy
29+
30+data Species = Species
31+ { speciesName :: Text
32+ , speciesGenus :: Text
33+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
34+
35+data Genus = Genus
36+ { genusName :: Text
37+ , genusFamily :: Text
38+ } deriving (Eq, Show, Read, Generic, ToJSON, FromJSON)
39+
40+instance FromRow Genus
41+instance FromRow Species
42+
43+server :: Connection -> Server API
44+server conn = ((\sname -> liftIO (lookupSpecies conn sname)
45+ :<|> liftIO (deleteSpecies conn sname))
46+ :<|> (\species -> liftIO $ insertSpecies conn species))
47+ :<|> (liftIO $ allSpecies conn)
48+
49+lookupSpecies :: Connection -> Text -> IO Species
50+lookupSpecies conn name = do
51+ [s] <- query conn "SELECT * FROM species WHERE species_name = ?" (Only name)
52+ return s
53+
54+deleteSpecies :: Connection -> Text -> IO ()
55+deleteSpecies conn name = do
56+ _ <- execute conn "DELETE FROM species WHERE species_name = ?" (Only name)
57+ return ()
58+
59+insertSpecies :: Connection -> Species -> IO ()
60+insertSpecies conn Species{..} = do
61+ _ <- execute conn "INSERT INTO species (species_name, species_genus) VALUES (?)" (speciesName, speciesGenus)
62+ return ()
63+
64+allSpecies :: Connection -> IO [Species]
65+allSpecies conn = do
66+ query_ conn "SELECT * FROM species"
67+
68+main :: IO ()
69+main = do
70+ conn <- connectPostgreSQL "dbname=servant-quickcheck"
71+ run 8090 (serve api $ server conn)
doc/posts/src/Spec.hsnew file+40
@@ -0,0 +1,40 @@
1+#line 304 "Announcement.anansi"
2+
3+#line 171 "Announcement.anansi"
4+
5+{-# LANGUAGE OverloadedStrings #-}
6+module Spec (main) where
7+
8+import Main (server, api, Species(..))
9+import Test.Hspec
10+import Test.QuickCheck.Instances
11+import Servant.QuickCheck
12+import Test.QuickCheck (Arbitrary(..))
13+import Database.PostgreSQL.Simple (connectPostgreSQL)
14+
15+spec :: Spec
16+spec = describe "the species application" $ beforeAll check $ do
17+ let pserver = do
18+ conn <- connectPostgreSQL "dbname=servant-quickcheck"
19+ return $ server conn
20+
21+
22+ it "should not return 500s" $ do
23+ withServantServer api pserver $ \url ->
24+ serverSatisfies api url defaultArgs (not500 <%> mempty)
25+
26+ it "should not return top-level json" $ do
27+ withServantServer api pserver $ \url ->
28+ serverSatisfies api url defaultArgs (onlyJsonObjects <%> mempty)
29+
30+ it "should return valid locations for 201" $ do
31+ withServantServer api pserver $ \url ->
32+ serverSatisfies api url defaultArgs (createContainsValidLocation <%> mempty)
33+
34+
35+main :: IO ()
36+main = do
37+ hspec spec
38+
39+instance Arbitrary Species where
40+ arbitrary = Species <$> arbitrary <*> arbitrary
doc/posts/src/schema.sqlnew file+9
@@ -0,0 +1,9 @@
1+CREATE TABLE genus (
2+ genus_name text PRIMARY KEY,
3+ genus_family text NOT NULL
4+);
5+
6+CREATE TABLE species (
7+ species_name text PRIMARY KEY,
8+ species_genus text NOT NULL REFERENCES genus (genus_name)
9+)
doc/posts/stack.yamlnew file+46
@@ -0,0 +1,46 @@
1+# This file was automatically generated by stack init
2+# For more information, see: http://docs.haskellstack.org/en/stable/yaml_configuration/
3+
4+
5+# Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2)
6+resolver: lts-5.14
7+
8+# Local packages, usually specified by relative directory name
9+packages:
10+- '../..'
11+- '.'
12+# Packages to be pulled from upstream that are not in the resolver (e.g., acme-missiles-0.3)
13+extra-deps:
14+- aeson-0.11.1.4
15+- fast-logger-2.4.6
16+- http2-1.6.0
17+- servant-0.7
18+- servant-server-0.7
19+- servant-client-0.7
20+- servant-quickcheck-0.1.0.0
21+- text-1.2.2.1
22+- warp-3.2.6
23+
24+# Override default flag values for local packages and extra-deps
25+flags: {}
26+
27+# Extra package databases containing global packages
28+extra-package-dbs: []
29+
30+# Control whether we use the GHC we find on the path
31+# system-ghc: true
32+
33+# Require a specific version of stack, using version ranges
34+# require-stack-version: -any # Default
35+# require-stack-version: >= 1.0.0
36+
37+# Override the architecture used by stack, especially useful on Windows
38+# arch: i386
39+# arch: x86_64
40+
41+# Extra directories used by stack for building
42+# extra-include-dirs: [/path/to/dir]
43+# extra-lib-dirs: [/path/to/dir]
44+
45+# Allow a newer minor version of GHC than the snapshot specifies
46+# compiler-check: newer-minor
example/Main.hsnew file+31
@@ -0,0 +1,31 @@
1+{-# LANGUAGE OverloadedStrings, DataKinds #-}
2+module Main (main) where
3+
4+import Servant
5+import Servant.QuickCheck
6+import Test.Hspec
7+import Data.Text (Text)
8+import System.Environment (getArgs)
9+
10+main :: IO ()
11+main = do
12+ args <- getArgs
13+ case args of
14+ [] -> putStrLn "Not running without arguments. Try --help or --fail-fast."
15+ _ -> hspec spec
16+
17+-- Change to String to reproduce
18+-- https://github.com/haskell-servant/servant-quickcheck/issues/41
19+type API = Get '[PlainText] Text
20+
21+api :: Proxy API
22+api = Proxy
23+
24+server :: Server API
25+server = return "Sigurð Fáfnirslayer"
26+
27+spec :: Spec
28+spec = describe "example server" $
29+ it "mangles UTF-8 in error messages" $
30+ withServantServer api (return server) $ \burl ->
31+ serverSatisfies api burl defaultArgs (getsHaveCacheControlHeader <%> mempty)
servant-quickcheck.cabal+126−89
@@ -1,101 +1,138 @@
1−name: servant-quickcheck
1+name: servant-quickcheck
2−version: 0.1.0.0
2+version: 0.0.9.1
3−synopsis: QuickCheck entire APIs
3+synopsis: QuickCheck entire APIs
4description:
4description:
5− This packages provides QuickCheck properties that are tested across an entire
5+ This packages provides QuickCheck properties that are tested across an entire
6− API.
6+ API.
7
7
8−license: BSD3
8+license: BSD3
9−license-file: LICENSE
9+license-file: LICENSE
10−author: Julian K. Arni
10+author: Julian K. Arni
11−maintainer: jkarni@gmail.com
11+maintainer: jkarni@gmail.com
12−category: Web
12+category: Web
13−build-type: Simple
13+build-type: Simple
14−cabal-version: >=1.10
14+cabal-version: >=1.10
15+extra-source-files: CHANGELOG.yaml
16+tested-with: GHC ==8.0.2 || ==8.2.2 || ==8.4.4 || ==8.6.5 || == 8.8.3
17+
18+source-repository head
19+ type: git
20+ location: https://github.com/haskell-servant/servant-quickcheck
15
21
16flag long-tests
22flag long-tests
17 description: Run more QuickCheck tests
23 description: Run more QuickCheck tests
18 default: False
24 default: False
19
25
20library
26library
27+ exposed-modules:
21− exposed-modules: Servant.QuickCheck
28+ Servant.QuickCheck
22− , Servant.QuickCheck.Internal
29+ Servant.QuickCheck.Internal
23− , Servant.QuickCheck.Internal.Benchmarking
30+ Servant.QuickCheck.Internal.Equality
31+ Servant.QuickCheck.Internal.ErrorTypes
32+ Servant.QuickCheck.Internal.HasGenRequest
24− , Servant.QuickCheck.Internal.Predicates
33+ Servant.QuickCheck.Internal.Predicates
25− , Servant.QuickCheck.Internal.HasGenRequest
26− , Servant.QuickCheck.Internal.QuickCheck
34+ Servant.QuickCheck.Internal.QuickCheck
27− , Servant.QuickCheck.Internal.Equality
35+
28− build-depends: base >=4.8 && <4.9
36+ build-depends:
29− , QuickCheck == 2.8.*
30− , bytestring == 0.10.*
31− , aeson > 0.10 && < 0.12
37+ aeson >=0.8 && <2
32− , mtl == 2.2.*
38+ , base >=4.9 && <4.15
33− , http-client == 0.4.*
39+ , base-compat-batteries >=0.10.1 && <0.12
34− , http-types == 0.9.*
40+ , bytestring >=0.10 && <0.11
41+ , case-insensitive >=1.2 && <1.3
42+ , clock >=0.7 && <0.9
43+ , data-default-class >=0.0 && <0.2
44+ , hspec >=2.5.6 && <2.8
45+ , http-client >=0.4.30 && <0.8
35− , http-media
46+ , http-media >=0.6 && <0.9
47+ , http-types >=0.8 && <0.13
48+ , mtl >=2.1 && <2.3
49+ , pretty >=1.1 && <1.2
50+ , process >=1.2 && <1.7
51+ , QuickCheck >=2.7 && <2.15
36− , servant-client == 0.7.*
52+ , servant >=0.17 && <0.19
53+ , servant-client >=0.17 && <0.19
37− , servant-server == 0.7.*
54+ , servant-server >=0.17 && <0.19
55+ , split >=0.2 && <0.3
38− , string-conversions == 0.4.*
56+ , string-conversions >=0.3 && <0.5
39− , data-default-class == 0.0.*
40− , servant == 0.7.*
41− , warp >= 3.2.4 && < 3.3
42− , process == 1.2.*
43− , temporary == 1.2.*
57+ , temporary >=1.2 && <1.4
44− , split == 0.2.*
45− , case-insensitive
46− , hspec
47− , text == 1.*
58+ , text >=1 && <2
59+ , time >=1.5 && <1.11
60+ , warp >=3.2.4 && <3.4
61+
62+ if !impl(ghc >=8.0)
63+ build-depends: semigroups >=0.18.3 && <0.20
64+
48− hs-source-dirs: src
65+ hs-source-dirs: src
49− default-extensions: TypeOperators
66+ default-extensions:
50− , FlexibleInstances
67+ NoImplicitPrelude
51− , FlexibleContexts
68+ ConstraintKinds
52− , DataKinds
69+ DataKinds
53− , GADTs
70+ DeriveDataTypeable
54− , MultiParamTypeClasses
55− , DeriveFunctor
71+ DeriveFunctor
72+ DeriveGeneric
73+ FlexibleContexts
74+ FlexibleInstances
75+ FunctionalDependencies
76+ GADTs
56− , KindSignatures
77+ KindSignatures
78+ MultiParamTypeClasses
79+ OverloadedStrings
57− , RankNTypes
80+ RankNTypes
58− , ConstraintKinds
59− , DeriveGeneric
60− , ScopedTypeVariables
81+ ScopedTypeVariables
61− , OverloadedStrings
82+ TypeOperators
62− , FunctionalDependencies
83+
63− default-language: Haskell2010
84+ default-language: Haskell2010
64
85
65test-suite spec
86test-suite spec
66− type: exitcode-stdio-1.0
87+ type: exitcode-stdio-1.0
67− ghc-options: -Wall -O2 -threaded
88+ ghc-options: -Wall -threaded
68− default-language: Haskell2010
89+ default-language: Haskell2010
69− hs-source-dirs: test
90+ hs-source-dirs: test
70− main-is: Spec.hs
91+ main-is: Spec.hs
71− other-modules: Servant.QuickCheck.InternalSpec
92+ other-modules: Servant.QuickCheck.InternalSpec
93+ build-tool-depends: hspec-discover:hspec-discover -any
72− build-depends: base == 4.*
94+ build-depends:
73− , servant-quickcheck
95+ aeson
96+ , base
97+ , base-compat-batteries
98+ , blaze-html
99+ , bytestring
100+ , hspec
74− , hspec
101+ , hspec-core >=2.5.5 && <2.8
75− , http-client
102+ , http-client
76− , warp
103+ , QuickCheck
104+ , quickcheck-io
105+ , servant
106+ , servant-blaze
107+ , servant-client
108+ , servant-quickcheck
77− , servant-server
109+ , servant-server
78− , servant-client
79− , servant
80− , transformers
110+ , transformers
81− , QuickCheck
111+ , warp
82− , quickcheck-io
112+
83− default-extensions: TypeOperators
113+ default-extensions:
84− , FlexibleInstances
114+ NoImplicitPrelude
115+ DataKinds
85− , FlexibleContexts
116+ FlexibleContexts
86− , DataKinds
117+ FlexibleInstances
118+ GADTs
119+ OverloadedStrings
120+ ScopedTypeVariables
121+ TypeOperators
122+
87 if flag(long-tests)
123 if flag(long-tests)
88− cpp-options: -DLONG_TESTS
124+ cpp-options: -DLONG_TESTS
125+
126+test-suite example
127+ type: exitcode-stdio-1.0
128+ main-is: Main.hs
129+ hs-source-dirs: example
130+ ghc-options: -Wall
131+ build-depends:
132+ base
133+ , hspec
134+ , servant-quickcheck
135+ , servant-server
136+ , text
89
137
90−-- test-suite doctests
91−-- default-language: Haskell2010
138+ default-language: Haskell2010
92−-- type: exitcode-stdio-1.0
93−-- ghc-options: -threaded
94−-- main-is: Doctest.hs
95−-- hs-source-dirs: test
96−-- build-depends: base >4 && <5
97−-- , doctest
98−-- , filemanip
99−-- , directory
100−-- , filepath
101−-- HS-Source-Dirs: test
src/Servant/QuickCheck.hs+53−17
@@ -6,14 +6,44 @@
6-- tested itself need not be implemented with @servant-server@ (or indeed,
6-- tested itself need not be implemented with @servant-server@ (or indeed,
7-- written in Haskell).
7-- written in Haskell).
8--
8--
9+-- The documentation of the <#useful-preds Useful predicates> sections is
10+-- meant to serve as a set of helpful pointers for learning more about best
11+-- practices concerning REST APIs.
9module Servant.QuickCheck
12module Servant.QuickCheck
10 (
13 (
11
14
15+ -- * Property testing
16+ serverSatisfies
17+ -- ** Predicates
18+ -- *** #useful-preds# Useful predicates
19+ -- | The predicates below are often useful. Some check RFC compliance; some are
20+ -- best practice, and some are useful to check that APIs follow in-house
21+ -- best-practices. Included in the documentation for each is a list of
22+ -- references to any relevant RFCs and other links, as well as what type of
23+ -- predicate it is (__RFC Compliance__, __Best Practice__, __Optional__).
24+ --
25+ -- RFCs distinguish between the force of requirements (e.g. __MUST__ vs.
26+ -- __SHOULD__). __RFC Compliance__ includes any absolute requirements present
27+ -- in RFCs. The __Best Practices__ includes, in addition to RFC
28+ -- recommendations, recommendations found elsewhere or generally accepted.
29+ , not500
30+ , notLongerThan
31+ , onlyJsonObjects
32+ , honoursAcceptHeader
33+ , notAllowedContainsAllowHeader
34+ , unauthorizedContainsWWWAuthenticate
35+ , getsHaveLastModifiedHeader
36+ , getsHaveCacheControlHeader
37+ , headsHaveCacheControlHeader
38+ , createContainsValidLocation
39+ -- * Html Predicates
40+ , htmlIncludesDoctype
12
41
13− -- * Test setup helpers
42+ -- *** Predicate utilities and types
14− -- | Helpers to setup and teardown @servant@ servers during tests.
43+ , (<%>)
15− withServantServer
44+ , Predicates
16−
45+ , ResponsePredicate(..)
46+ , RequestPredicate(..)
17
47
18 -- * Equality testing
48 -- * Equality testing
19 , serversEqual
49 , serversEqual
@@ -24,29 +54,35 @@ module Servant.QuickCheck
24 -- represents other means of checking equality
54 -- represents other means of checking equality
25 -- *** Useful @ResponseEquality@s
55 -- *** Useful @ResponseEquality@s
26 , bodyEquality
56 , bodyEquality
57+ , jsonEquality
27 , allEquality
58 , allEquality
28 -- ** Response equality type
59 -- ** Response equality type
29 , ResponseEquality(..)
60 , ResponseEquality(..)
30
61
31− -- * Property testing
62+ -- * Test setup helpers
32− , serverSatisfies
63+ -- | Helpers to setup and teardown @servant@ servers during tests.
33− -- ** Predicates
64+ , withServantServer
34− -- *** Useful predicates
65+ , withServantServerAndContext
35− , not500
66+ , defaultArgs
36− , onlyJsonObjects
37− , notAllowedContainsAllowHeader
38− -- *** Predicate utilities and types
39− , (<%>)
40− , Predicates
41− , ResponsePredicate(..)
42− , RequestPredicate(..)
43
67
44 -- ** Re-exports
68 -- ** Re-exports
69+ -- | Types and constructors from other packages that are generally needed for
70+ -- using @servant-quickcheck@.
45 , BaseUrl(..)
71 , BaseUrl(..)
46 , Scheme(..)
72 , Scheme(..)
73+ , Args(..)
74+ , Proxy(..)
47
75
48
76
49 ) where
77 ) where
50
78
79+import Data.Proxy (Proxy (..))
80+import Servant.Client (BaseUrl (..), Scheme (..))
51import Servant.QuickCheck.Internal
81import Servant.QuickCheck.Internal
52−import Servant.Client (BaseUrl(..), Scheme(..))
82+import Test.QuickCheck (Args (..), stdArgs)
83+
84+-- | QuickCheck @Args@ with 1000 rather than 100 test cases.
85+--
86+-- /Since 0.0.0.0/
87+defaultArgs :: Args
88+defaultArgs = stdArgs { maxSuccess = 1000 }
src/Servant/QuickCheck/Internal.hs+4−4
@@ -1,7 +1,7 @@
1module Servant.QuickCheck.Internal (module X) where
1module Servant.QuickCheck.Internal (module X) where
2
2
3+import Servant.QuickCheck.Internal.Equality as X
4+import Servant.QuickCheck.Internal.ErrorTypes as X
3import Servant.QuickCheck.Internal.HasGenRequest as X
5import Servant.QuickCheck.Internal.HasGenRequest as X
4−import Servant.QuickCheck.Internal.Predicates as X
6+import Servant.QuickCheck.Internal.Predicates as X
5−import Servant.QuickCheck.Internal.QuickCheck as X
7+import Servant.QuickCheck.Internal.QuickCheck as X
6−import Servant.QuickCheck.Internal.Equality as X
7−import Servant.QuickCheck.Internal.Benchmarking as X
src/Servant/QuickCheck/Internal/Benchmarking.hsdeleted−87
@@ -1,87 +0,0 @@
1−-- | This module contains benchmark-related logic.
2−--
3−-- Currently it generates 'wrk' scripts rather than benchmarking directly with
4−-- the @servant-client@ functions since the performance of 'wrk' is
5−-- significantly better.
6−module Servant.QuickCheck.Internal.Benchmarking where
7−
8−import Data.ByteString (ByteString)
9−import Data.ByteString.Lazy (toStrict)
10−import Network.HTTP.Client
11−import Network.HTTP.Types
12−import Servant.Client
13−
14−data BenchOptions = BenchOptions
15− { duration :: Int
16− , threads :: Int
17− , connections :: Int
18− , noOfTests :: Int
19− } deriving (Eq, Show, Read)
20−
21−defaultBenchOptions :: BenchOptions
22−defaultBenchOptions = BenchOptions
23− { duration = 10
24− , threads = 1
25− , connections = 10
26− , noOfTests = 10
27− }
28−
29−data WrkScript = WrkScript
30− { wrkScheme :: Scheme
31− , wrkHost :: ByteString
32− , wrkPort :: Int
33− , wrkMethod :: Method
34− , wrkPath :: ByteString
35− , wrkHeaders :: [Header]
36− , wrkBody :: ByteString
37− } deriving (Eq, Show)
38−
39−mkScript :: WrkScript -> String
40−mkScript w
41− = "wrk.scheme = \"" ++ sscheme (wrkScheme w) ++ "\""
42− ++ "\nwrk.host = " ++ show (wrkHost w)
43− ++ "\nwrk.port = " ++ show (wrkPort w)
44− ++ "\nwrk.method = " ++ show (wrkMethod w)
45− ++ "\nwrk.path = " ++ show (wrkPath w)
46− ++ foldr (\(h,v) old -> old ++ "\nwrk.headers[" ++ show h ++ "] = " ++ show v)
47− ""
48− (wrkHeaders w)
49− ++ "\nwrk.body = " ++ show (wrkBody w)
50− ++ "\n" ++ reportFmt
51− where
52− sscheme Http = "http"
53− sscheme Https = "https"
54−
55−reqToWrk :: Request -> WrkScript
56−reqToWrk r = WrkScript
57− { wrkScheme = Http
58− , wrkHost = host r
59− , wrkPort = port r
60− , wrkMethod = method r
61− , wrkPath = path r
62− , wrkHeaders = requestHeaders r
63− , wrkBody = case requestBody r of
64− RequestBodyLBS r' -> toStrict r'
65− _ -> error "expecting RequestBodyLBS"
66− }
67−
68−reportFmt :: String
69−reportFmt
70− = "done = function(summary, latency, requests)\n"
71− ++ " for _, p in pairs({ 50, 75, 99, 99.999 }) do\n"
72− ++ " n = latency:percentile(p)\n"
73− ++ " io.write(string.format(\"%g%%, %d\\n\", p, n))\n"
74− ++ " end\n"
75− ++ "end\n"
76−
77−{-data BenchResult = BenchResult-}
78− {-{ benchReq :: Request-}
79− {-, benchLatencyDist :: [(Percentile, Microsecs)]-}
80− {-, benchLatencyAvg :: Microsecs-}
81− {-} deriving (Eq, Show, Read, Generic)-}
82−
83−{-newtype Microsecs = Microsecs { unMicroSecs :: Int }-}
84− {-deriving (Eq, Show, Read, Generic)-}
85−
86−{-newtype Percentile = Percentile { unPercentile :: Int }-}
87− {-deriving (Eq, Show, Read, Generic)-}
src/Servant/QuickCheck/Internal/Equality.hs+42−4
@@ -1,20 +1,58 @@
1module Servant.QuickCheck.Internal.Equality where
1module Servant.QuickCheck.Internal.Equality where
2
2
3+import Data.Aeson (Value, decode, decodeStrict)
4+import Data.ByteString (ByteString)
5+import qualified Data.ByteString.Lazy as LB
6+import Data.Function (on)
3−import Network.HTTP.Client
7+import Network.HTTP.Client (Response, responseBody)
4−import Data.Function (on)
8+import Data.Semigroup (Semigroup (..))
9+import Prelude.Compat
5
10
6newtype ResponseEquality b
11newtype ResponseEquality b
7 = ResponseEquality { getResponseEquality :: Response b -> Response b -> Bool }
12 = ResponseEquality { getResponseEquality :: Response b -> Response b -> Bool }
8
13
14+instance Semigroup (ResponseEquality b) where
15+ ResponseEquality a <> ResponseEquality b = ResponseEquality $ \x y ->
16+ a x y && b x y
17+
9instance Monoid (ResponseEquality b) where
18instance Monoid (ResponseEquality b) where
10 mempty = ResponseEquality $ \_ _ -> True
19 mempty = ResponseEquality $ \_ _ -> True
11− ResponseEquality a `mappend` ResponseEquality b = ResponseEquality $ \x y ->
20+ mappend = (<>)
12− a x y && b x y
13
21
14-- | Use `Eq` instance for `Response`
22-- | Use `Eq` instance for `Response`
23+--
24+-- /Since 0.0.0.0/
15allEquality :: Eq b => ResponseEquality b
25allEquality :: Eq b => ResponseEquality b
16allEquality = ResponseEquality (==)
26allEquality = ResponseEquality (==)
17
27
18-- | ByteString `Eq` instance over the response body.
28-- | ByteString `Eq` instance over the response body.
29+--
30+-- /Since 0.0.0.0/
19bodyEquality :: Eq b => ResponseEquality b
31bodyEquality :: Eq b => ResponseEquality b
20bodyEquality = ResponseEquality ((==) `on` responseBody)
32bodyEquality = ResponseEquality ((==) `on` responseBody)
33+
34+-- | Equality as 'Value'. This means that if two bodies are equal as JSON
35+-- (e.g., insignificant whitespace difference) they are considered equal.
36+--
37+-- /Since 0.0.3.0/
38+jsonEquality :: (JsonEq b) => ResponseEquality b
39+jsonEquality = ResponseEquality (jsonEq `on` responseBody)
40+
41+class JsonEq a where
42+ decode' :: a -> Maybe Value
43+ jsonEq :: a -> a -> Bool
44+ jsonEq first second = compareDecodedResponses (decode' first) (decode' second)
45+
46+instance JsonEq LB.ByteString where
47+ decode' = decode
48+
49+instance JsonEq ByteString where
50+ decode' = decodeStrict
51+
52+compareDecodedResponses :: Maybe Value -> Maybe Value -> Bool
53+compareDecodedResponses resp1 resp2 =
54+ case resp1 of
55+ Nothing -> False -- if decoding fails we assume failure
56+ (Just r1) -> case resp2 of
57+ Nothing -> False -- another decode failure
58+ (Just r2) -> r1 == r2
src/Servant/QuickCheck/Internal/ErrorTypes.hsnew file+82
@@ -0,0 +1,82 @@
1+{-# LANGUAGE CPP #-}
2+module Servant.QuickCheck.Internal.ErrorTypes where
3+
4+import Control.Exception (Exception (..))
5+import qualified Data.ByteString.Lazy as LBS
6+import Data.String.Conversions (cs)
7+import qualified Data.Text as T
8+import Data.Typeable (Typeable)
9+import GHC.Generics (Generic)
10+import qualified Network.HTTP.Client as C
11+import Network.HTTP.Types (Header, statusCode)
12+import Text.PrettyPrint
13+
14+import Prelude.Compat hiding ((<>))
15+
16+data PredicateFailure
17+ = PredicateFailure T.Text (Maybe C.Request) (C.Response LBS.ByteString)
18+ deriving (Typeable, Generic)
19+
20+instance Exception ServerEqualityFailure where
21+
22+instance Show PredicateFailure where
23+ show = render . prettyPredicateFailure
24+
25+
26+data ServerEqualityFailure
27+ = ServerEqualityFailure C.Request (C.Response LBS.ByteString) (C.Response LBS.ByteString)
28+ deriving (Typeable, Generic)
29+
30+instance Show ServerEqualityFailure where
31+ show = render . prettyServerEqualityFailure
32+
33+
34+instance Exception PredicateFailure where
35+
36+-- * Pretty printing
37+
38+prettyHeaders :: [Header] -> Doc
39+prettyHeaders hdrs = vcat $ prettyHdr <$> hdrs
40+ where
41+ prettyHdr (hn, h) = text (show hn) <> colon <+> text (show h)
42+
43+prettyReq :: C.Request -> Doc
44+prettyReq r =
45+ text "Request:" $$ (nest 5 $
46+ text "Method:" <+> (nest 5 $ text . show $ C.method r)
47+ $$ text "Path:" <+> (nest 5 $ text . cs $ C.path r)
48+ $$ text "Headers:" <+> (nest 5 $ prettyHeaders $ C.requestHeaders r)
49+ $$ text "Body:" <+> (nest 5 $ text . getReqBody $ C.requestBody r))
50+ where
51+ getReqBody (C.RequestBodyLBS lbs ) = cs lbs
52+ getReqBody (C.RequestBodyBS bs ) = cs bs
53+ getReqBody _ = error "expected bytestring body"
54+
55+prettyResp :: C.Response LBS.ByteString -> Doc
56+prettyResp r =
57+ text "Response:" $$ (nest 5 $
58+ text "Status code:" <+> (nest 5 $ text . show . statusCode $ C.responseStatus r)
59+ $$ text "Headers:" $$ (nest 10 $ prettyHeaders $ C.responseHeaders r)
60+ $$ text "Body:" <+> (nest 5 $ text . cs $ C.responseBody r))
61+
62+
63+
64+prettyServerEqualityFailure :: ServerEqualityFailure -> Doc
65+prettyServerEqualityFailure (ServerEqualityFailure req resp1 resp2) =
66+ text "Server equality failed" $$ (nest 5 $
67+ prettyReq req
68+ $$ prettyResp resp1
69+ $$ prettyResp resp2)
70+
71+
72+prettyPredicateFailure :: PredicateFailure -> Doc
73+prettyPredicateFailure (PredicateFailure predicate req resp) =
74+ text "Predicate failed" $$ (nest 5 $
75+ text "Predicate:" <+> (text $ T.unpack predicate)
76+ $$ r
77+ $$ prettyResp resp)
78+ where
79+ r = case req of
80+ Nothing -> text ""
81+ Just v -> prettyReq v
82+
src/Servant/QuickCheck/Internal/HasGenRequest.hs+117−52
@@ -1,97 +1,148 @@
1+{-# LANGUAGE CPP #-}
1{-# LANGUAGE PolyKinds #-}
2{-# LANGUAGE PolyKinds #-}
2module Servant.QuickCheck.Internal.HasGenRequest where
3module Servant.QuickCheck.Internal.HasGenRequest where
3
4
4−import Data.Default.Class (def)
5−import Data.Monoid ((<>))
6−import Data.String (fromString)
5+import Data.String (fromString)
7−import Data.String.Conversions (cs)
6+import Data.String.Conversions (cs)
8−import GHC.TypeLits
7+import GHC.TypeLits (KnownSymbol, Nat, symbolVal)
9−import Network.HTTP.Client (Request, RequestBody (..), host,
8+import Network.HTTP.Client (Request, RequestBody (..), host, method, path,
10− method, path, port, requestBody,
9+ port, queryString, requestBody, requestHeaders,
11− requestHeaders, secure, queryString)
10+ secure, defaultRequest)
12−import Network.HTTP.Media (renderHeader)
11+import Network.HTTP.Media (renderHeader)
12+import Prelude.Compat
13−import Servant
13+import Servant
14−import Servant.API.ContentTypes
14+import Servant.API.ContentTypes (AllMimeRender (..))
15−import Servant.Client (BaseUrl (..), Scheme (..))
15+import Servant.Client (BaseUrl (..), Scheme (..))
16−import Test.QuickCheck
16+import Test.QuickCheck (Arbitrary (..), Gen, elements, frequency)
17−
17+
18−
18+import qualified Data.ByteString as BS
19+import qualified Data.ByteString.Internal as BS (c2w)
20+
21+
22+-- -----------------------------------------------------------------------------
23+-- runGenRequest
24+
25+-- | This function returns a QuickCheck `Gen a` when passed a servant API value,
26+-- typically a `Proxy API`. The generator returned is a function
27+-- that accepts a `BaseUrl` and returns a `Request`, which can then be used
28+-- to issue network requests. This `Gen` type makes it easier to compare distinct
29+-- APIs across different `BaseUrl`s.
30+runGenRequest :: HasGenRequest a => Proxy a -> Gen (BaseUrl -> Request)
31+runGenRequest = snd . genRequest
32+
33+
34+-- -----------------------------------------------------------------------------
35+-- HasGenRequest
36+
37+-- | This is the core Servant-Quickcheck generator, which, when given a `Proxy API`
38+-- will return a pair of `Int` and `Gen a`, where `a` is a function from
39+-- `BaseUrl` to a `Network.Http.Client.Request`. The `Int` is a weight for the
40+-- QuickCheck `frequency` function which ensures a random distribution across
41+-- all endpoints in an API.
19class HasGenRequest a where
42class HasGenRequest a where
20− genRequest :: Proxy a -> Gen (BaseUrl -> Request)
43+ genRequest :: Proxy a -> (Int, Gen (BaseUrl -> Request))
44+
21
45
22instance (HasGenRequest a, HasGenRequest b) => HasGenRequest (a :<|> b) where
46instance (HasGenRequest a, HasGenRequest b) => HasGenRequest (a :<|> b) where
23 genRequest _
47 genRequest _
48+ = (lf + rf, frequency [l, r])
49+ where
24− = oneof [ genRequest (Proxy :: Proxy a)
50+ l@(lf, _) = genRequest (Proxy :: Proxy a)
25− , genRequest (Proxy :: Proxy b)
51+ r@(rf, _) = genRequest (Proxy :: Proxy b)
26− ]
52+
27
53
28instance (KnownSymbol path, HasGenRequest b) => HasGenRequest (path :> b) where
54instance (KnownSymbol path, HasGenRequest b) => HasGenRequest (path :> b) where
29− genRequest _ = do
55+ genRequest _ = (oldf, do
30 old' <- old
56 old' <- old
31− return $ \burl -> let r = old' burl in r { path = new <> "/" <> path r }
57+ return $ \burl -> let r = old' burl
58+ oldPath = path r
59+ oldPath' = BS.dropWhile (== BS.c2w '/') oldPath
60+ paths = filter (not . BS.null) [new, oldPath']
61+ in r { path = "/" <> BS.intercalate "/" paths })
32 where
62 where
33− old = genRequest (Proxy :: Proxy b)
63+ (oldf, old) = genRequest (Proxy :: Proxy b)
34 new = cs $ symbolVal (Proxy :: Proxy path)
64 new = cs $ symbolVal (Proxy :: Proxy path)
35
65
66+instance HasGenRequest EmptyAPI where
67+ genRequest _ = (0, error "EmptyAPIs cannot be queried.")
68+
69+instance HasGenRequest api => HasGenRequest (Summary d :> api) where
70+ genRequest _ = genRequest (Proxy :: Proxy api)
71+
72+instance HasGenRequest api => HasGenRequest (Description d :> api) where
73+ genRequest _ = genRequest (Proxy :: Proxy api)
74+
36instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
75instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
37− => HasGenRequest (Capture x c :> b) where
76+ => HasGenRequest (Capture' mods x c :> b) where
38− genRequest _ = do
77+ genRequest _ = (oldf, do
39 old' <- old
78 old' <- old
40 new' <- toUrlPiece <$> new
79 new' <- toUrlPiece <$> new
41− return $ \burl -> let r = old' burl in r { path = cs new' <> "/" <> path r }
80+ return $ \burl -> let r = old' burl in r { path = cs new' <> path r })
42 where
81 where
43− old = genRequest (Proxy :: Proxy b)
82+ (oldf, old) = genRequest (Proxy :: Proxy b)
44 new = arbitrary :: Gen c
83 new = arbitrary :: Gen c
45
84
85+instance (Arbitrary c, HasGenRequest b, ToHttpApiData c )
86+ => HasGenRequest (CaptureAll x c :> b) where
87+ genRequest _ = (oldf, do
88+ old' <- old
89+ new' <- fmap (cs . toUrlPiece) <$> new
90+ let new'' = BS.intercalate "/" new'
91+ return $ \burl -> let r = old' burl in r { path = new'' <> path r })
92+ where
93+ (oldf, old) = genRequest (Proxy :: Proxy b)
94+ new = arbitrary :: Gen [c]
95+
46instance (Arbitrary c, KnownSymbol h, HasGenRequest b, ToHttpApiData c)
96instance (Arbitrary c, KnownSymbol h, HasGenRequest b, ToHttpApiData c)
47− => HasGenRequest (Header h c :> b) where
97+ => HasGenRequest (Header' mods h c :> b) where
48− genRequest _ = do
98+ genRequest _ = (oldf, do
49 old' <- old
99 old' <- old
50− new' <- toUrlPiece <$> new
100+ new' <- toUrlPiece <$> new -- TODO: generate lenient or/and optional
51 return $ \burl -> let r = old' burl in r {
101 return $ \burl -> let r = old' burl in r {
52− requestHeaders = (hdr, cs new') : requestHeaders r }
102+ requestHeaders = (hdr, cs new') : requestHeaders r })
53 where
103 where
54− old = genRequest (Proxy :: Proxy b)
104+ (oldf, old) = genRequest (Proxy :: Proxy b)
55 hdr = fromString $ symbolVal (Proxy :: Proxy h)
105 hdr = fromString $ symbolVal (Proxy :: Proxy h)
56 new = arbitrary :: Gen c
106 new = arbitrary :: Gen c
57
107
58instance (AllMimeRender x c, Arbitrary c, HasGenRequest b)
108instance (AllMimeRender x c, Arbitrary c, HasGenRequest b)
59− => HasGenRequest (ReqBody x c :> b) where
109+ => HasGenRequest (ReqBody' mods x c :> b) where
60− genRequest _ = do
110+ genRequest _ = (oldf, do
61− old' <- old
111+ old' <- old -- TODO: generate lenient
62 new' <- new
112 new' <- new
63 (ct, bd) <- elements $ allMimeRender (Proxy :: Proxy x) new'
113 (ct, bd) <- elements $ allMimeRender (Proxy :: Proxy x) new'
64 return $ \burl -> let r = old' burl in r {
114 return $ \burl -> let r = old' burl in r {
65 requestBody = RequestBodyLBS bd
115 requestBody = RequestBodyLBS bd
66 , requestHeaders = ("Content-Type", renderHeader ct) : requestHeaders r
116 , requestHeaders = ("Content-Type", renderHeader ct) : requestHeaders r
67− }
117+ })
68 where
118 where
69− old = genRequest (Proxy :: Proxy b)
119+ (oldf, old) = genRequest (Proxy :: Proxy b)
70 new = arbitrary :: Gen c
120 new = arbitrary :: Gen c
71
121
72instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
122instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
73− => HasGenRequest (QueryParam x c :> b) where
123+ => HasGenRequest (QueryParam' mods x c :> b) where
74− genRequest _ = do
124+ genRequest _ = (oldf, do
75− new' <- new
125+ new' <- new -- TODO: generate lenient or/and optional
76 old' <- old
126 old' <- old
77− return $ \burl -> let r = old' burl in r {
127+ return $ \burl -> let r = old' burl
78− queryString = queryString r
79− <> param <> "=" <> cs (toQueryParam new') }
128+ newExpr = param <> "=" <> cs (toQueryParam new')
129+ qs = queryString r in r {
130+ queryString = if BS.null qs then newExpr else newExpr <> "&" <> qs })
80 where
131 where
81− old = genRequest (Proxy :: Proxy b)
132+ (oldf, old) = genRequest (Proxy :: Proxy b)
82 param = cs $ symbolVal (Proxy :: Proxy x)
133 param = cs $ symbolVal (Proxy :: Proxy x)
83 new = arbitrary :: Gen c
134 new = arbitrary :: Gen c
84
135
85instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
136instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
86 => HasGenRequest (QueryParams x c :> b) where
137 => HasGenRequest (QueryParams x c :> b) where
87− genRequest _ = do
138+ genRequest _ = (oldf, do
88 new' <- new
139 new' <- new
89 old' <- old
140 old' <- old
90 return $ \burl -> let r = old' burl in r {
141 return $ \burl -> let r = old' burl in r {
91 queryString = queryString r
142 queryString = queryString r
92− <> if length new' > 0 then fold (toParam <$> new') else ""}
143+ <> if length new' > 0 then fold (toParam <$> new') else ""})
93 where
144 where
94− old = genRequest (Proxy :: Proxy b)
145+ (oldf, old) = genRequest (Proxy :: Proxy b)
95 param = cs $ symbolVal (Proxy :: Proxy x)
146 param = cs $ symbolVal (Proxy :: Proxy x)
96 new = arbitrary :: Gen [c]
147 new = arbitrary :: Gen [c]
97 toParam c = param <> "[]=" <> cs (toQueryParam c)
148 toParam c = param <> "[]=" <> cs (toQueryParam c)
@@ -99,22 +150,32 @@ instance (KnownSymbol x, Arbitrary c, ToHttpApiData c, HasGenRequest b)
99
150
100instance (KnownSymbol x, HasGenRequest b)
151instance (KnownSymbol x, HasGenRequest b)
101 => HasGenRequest (QueryFlag x :> b) where
152 => HasGenRequest (QueryFlag x :> b) where
102− genRequest _ = do
153+ genRequest _ = (oldf, do
103 old' <- old
154 old' <- old
104− return $ \burl -> let r = old' burl in r {
155+ return $ \burl -> let r = old' burl
156+ qs = queryString r in r {
105− queryString = queryString r <> param <> "=" }
157+ queryString = if BS.null qs then param else param <> "&" <> qs })
106 where
158 where
107− old = genRequest (Proxy :: Proxy b)
159+ (oldf, old) = genRequest (Proxy :: Proxy b)
108 param = cs $ symbolVal (Proxy :: Proxy x)
160 param = cs $ symbolVal (Proxy :: Proxy x)
109
161
110instance (ReflectMethod method)
162instance (ReflectMethod method)
111 => HasGenRequest (Verb (method :: k) (status :: Nat) (cts :: [*]) a) where
163 => HasGenRequest (Verb (method :: k) (status :: Nat) (cts :: [*]) a) where
112− genRequest _ = return $ \burl -> def
164+ genRequest _ = (1, return $ \burl -> defaultRequest
113 { host = cs $ baseUrlHost burl
165 { host = cs $ baseUrlHost burl
114 , port = baseUrlPort burl
166 , port = baseUrlPort burl
115 , secure = baseUrlScheme burl == Https
167 , secure = baseUrlScheme burl == Https
116 , method = reflectMethod (Proxy :: Proxy method)
168 , method = reflectMethod (Proxy :: Proxy method)
117− }
169+ })
170+
171+instance (ReflectMethod method)
172+ => HasGenRequest (NoContentVerb (method :: k)) where
173+ genRequest _ = (1, return $ \burl -> defaultRequest
174+ { host = cs $ baseUrlHost burl
175+ , port = baseUrlPort burl
176+ , secure = baseUrlScheme burl == Https
177+ , method = reflectMethod (Proxy :: Proxy method)
178+ })
118
179
119instance (HasGenRequest a) => HasGenRequest (RemoteHost :> a) where
180instance (HasGenRequest a) => HasGenRequest (RemoteHost :> a) where
120 genRequest _ = genRequest (Proxy :: Proxy a)
181 genRequest _ = genRequest (Proxy :: Proxy a)
@@ -130,3 +191,7 @@ instance (HasGenRequest a) => HasGenRequest (Vault :> a) where
130
191
131instance (HasGenRequest a) => HasGenRequest (WithNamedContext x y a) where
192instance (HasGenRequest a) => HasGenRequest (WithNamedContext x y a) where
132 genRequest _ = genRequest (Proxy :: Proxy a)
193 genRequest _ = genRequest (Proxy :: Proxy a)
194+
195+-- TODO: Try logging in
196+instance (HasGenRequest a) => HasGenRequest (BasicAuth x y :> a) where
197+ genRequest _ = genRequest (Proxy :: Proxy a)
src/Servant/QuickCheck/Internal/Predicates.hs+323−123
@@ -1,30 +1,69 @@
1module Servant.QuickCheck.Internal.Predicates where
1module Servant.QuickCheck.Internal.Predicates where
2
2
3−import Control.Monad
3+import Control.Exception (catch, throw)
4+import Control.Monad (liftM2, unless, when)
4−import Data.Aeson (Object, decode)
5+import Data.Aeson (Object, decode)
5−import Data.Bifunctor (Bifunctor (..))
6+import Data.Bifunctor (first)
6−import qualified Data.ByteString.Lazy as LBS
7+import qualified Data.ByteString as SBS
7−import qualified Data.ByteString as SBS
8import qualified Data.ByteString.Char8 as SBSC
8import qualified Data.ByteString.Char8 as SBSC
9+import qualified Data.ByteString.Lazy as LBS
9−import Data.CaseInsensitive (mk)
10+import Data.CaseInsensitive (foldCase, foldedCase, mk)
10−import Data.Either (isRight)
11+import Data.Either (isRight)
11−import Data.List.Split (wordsBy)
12+import Data.List.Split (wordsBy)
12−import Data.Monoid ((<>))
13+import Data.Maybe (fromMaybe, isJust)
13−import Data.Text (Text)
14+import Data.Semigroup (Semigroup (..))
15+import qualified Data.Text as T
16+import Data.Time (UTCTime, defaultTimeLocale, parseTimeM,
17+ rfc822DateFormat)
14−import GHC.Generics (Generic)
18+import GHC.Generics (Generic)
15−import Network.HTTP.Client (Manager, Request, Response, httpLbs,
19+import Network.HTTP.Client (Manager, Request, Response, httpLbs,
20+ method, parseRequest, requestHeaders,
21+ responseBody, responseHeaders,
16− responseBody, responseStatus, responseHeaders)
22+ responseStatus)
17−import Network.HTTP.Types (status500, status405, parseMethod)
23+import Network.HTTP.Media (matchAccept)
18−
24+import Network.HTTP.Types (methodGet, methodHead, parseMethod,
25+ renderStdMethod, status100, status200,
26+ status201, status300, status401,
27+ status405, status500)
28+import Prelude.Compat
29+import System.Clock (Clock (Monotonic), diffTimeSpec,
30+ getTime, toNanoSecs)
31+
32+import Servant.QuickCheck.Internal.ErrorTypes
33+
34+
35+-- | [__Best Practice__]
36+--
19−-- | @500 Internal Server Error@ should be avoided - it may represent some
37+-- @500 Internal Server Error@ should be avoided - it may represent some
20-- issue with the application code, and it moreover gives the client little
38-- issue with the application code, and it moreover gives the client little
21-- indication of how to proceed or what went wrong.
39-- indication of how to proceed or what went wrong.
22--
40--
23-- This function checks that the response code is not 500.
41-- This function checks that the response code is not 500.
42+--
43+-- /Since 0.0.0.0/
24−not500 :: ResponsePredicate Text Bool
44+not500 :: ResponsePredicate
25−not500 = ResponsePredicate "not500" (\resp -> not $ responseStatus resp == status500)
45+not500 = ResponsePredicate $ \resp ->
46+ when (responseStatus resp == status500) $ throw $ PredicateFailure "not500" Nothing resp
26
47
48+-- | [__Optional__]
49+--
50+-- This function checks that the response from the server does not take longer
51+-- than the specified number of nanoseconds.
52+--
53+-- /Since 0.0.2.1/
54+notLongerThan :: Integer -> RequestPredicate
55+notLongerThan maxAllowed
56+ = RequestPredicate $ \req mgr -> do
57+ start <- getTime Monotonic
58+ resp <- httpLbs req mgr
59+ end <- getTime Monotonic
60+ when (toNanoSecs (end `diffTimeSpec` start) > maxAllowed) $
61+ throw $ PredicateFailure "notLongerThan" (Just req) resp
62+ return []
63+
64+-- | [__Best Practice__]
65+--
27−-- | Returning anything other than an object when returning JSON is considered
66+-- Returning anything other than an object when returning JSON is considered
28-- bad practice, as:
67-- bad practice, as:
29--
68--
30-- (1) it is hard to modify the returned value while maintaining backwards
69-- (1) it is hard to modify the returned value while maintaining backwards
@@ -41,79 +80,197 @@ not500 = ResponsePredicate "not500" (\resp -> not $ responseStatus resp == statu
41--
80--
42-- * JSON Grammar: <https://tools.ietf.org/html/rfc7159#section-2 RFC 7159 Section 2>
81-- * JSON Grammar: <https://tools.ietf.org/html/rfc7159#section-2 RFC 7159 Section 2>
43-- * JSON Grammar: <https://tools.ietf.org/html/rfc4627#section-2 RFC 4627 Section 2>
82-- * JSON Grammar: <https://tools.ietf.org/html/rfc4627#section-2 RFC 4627 Section 2>
83+--
84+-- /Since 0.0.0.0/
44−onlyJsonObjects :: ResponsePredicate Text Bool
85+onlyJsonObjects :: ResponsePredicate
45onlyJsonObjects
86onlyJsonObjects
87+ = ResponsePredicate (\resp -> do
88+ case lookup "content-type" (first foldedCase <$> responseHeaders resp) of
89+ Nothing -> return ()
90+ Just ctype -> when ("application/json" `SBS.isPrefixOf` ctype) $ do
46− = ResponsePredicate "onlyJsonObjects" (\resp -> case decode (responseBody resp) of
91+ case (decode (responseBody resp) :: Maybe Object) of
47− Nothing -> False
92+ Nothing -> throw $ PredicateFailure "onlyJsonObjects" Nothing resp
48− Just (_ :: Object) -> True)
93+ Just _ -> return ())
49−
94+
50−{-
95+-- | __Optional__
96+--
51−-- | When creating a new resource, it is good practice to provide a @Location@
97+-- When creating a new resource, it is good practice to provide a @Location@
52-- header with a link to the created resource.
98-- header with a link to the created resource.
53--
99--
54-- This function checks that every @201 Created@ response contains a @Location@
100-- This function checks that every @201 Created@ response contains a @Location@
55-- header, and that the link in it responds with a 2XX response code to @GET@
101-- header, and that the link in it responds with a 2XX response code to @GET@
56-- requests.
102-- requests.
57--
103--
104+-- This is considered optional because other means of linking to the resource
105+-- (e.g. via the response body) are also acceptable; linking to the resource in
106+-- some way is considered best practice.
107+--
108+-- __References__:
109+--
110+-- * 201 Created: <https://tools.ietf.org/html/rfc7231#section-6.3.2 RFC 7231 Section 6.3.2>
58−-- References: <RFC 7231, Section 6.3.2 https://tools.ietf.org/html/rfc7231#section-6.3.2>
111+-- * Location header: <https://tools.ietf.org/html/rfc7231#section-7.1.2 RFC 7231 Section 7.1.2>
112+--
113+-- /Since 0.0.0.0/
59−createContainsValidLocation :: ResponsePredicate Text Bool
114+createContainsValidLocation :: RequestPredicate
60createContainsValidLocation
115createContainsValidLocation
116+ = RequestPredicate $ \req mgr -> do
61− = ResponsePredicate "createContainsValidLocation" (\resp ->
117+ let n = "createContainsValidLocation"
62−
118+ resp <- httpLbs req mgr
119+ if responseStatus resp == status201
120+ then case lookup "Location" $ responseHeaders resp of
121+ Nothing -> throw $ PredicateFailure n (Just req) resp
122+ Just l -> case parseRequest $ SBSC.unpack l of
123+ Nothing -> throw $ PredicateFailure n (Just req) resp
124+ Just x -> do
125+ resp2 <- httpLbs x mgr
126+ status2XX (Just req) resp2 n
127+ return [resp, resp2]
128+ else return [resp]
129+
130+-- | [__Optional__]
131+--
132+-- The @Last-Modified@ header represents the time a resource was last
133+-- modified. It is used to drive caching and conditional requests.
134+--
135+-- When using this mechanism, the server adds the @Last-Modified@ header to
136+-- responses. Clients may then make requests with the @If-Modified-Since@
137+-- header to conditionally request resources. If the resource has not
138+-- changed since that date, the server responds with a status code of 304
139+-- (@Not Modified@) without a response body.
140+--
141+-- The @Last-Modified@ header can also be used in conjunction with the
142+-- @If-Unmodified-Since@ header to drive optimistic concurrency.
143+--
144+-- The @Last-Modified@ date must be in RFC 822 format.
145+--
146+-- __References__:
147+--
148+-- * 304 Not Modified: <https://tools.ietf.org/html/rfc7232#section-4.1 RFC 7232 Section 4.1>
149+-- * Last-Modified header: <https://tools.ietf.org/html/rfc7232#section-2.2 RFC 7232 Section 2.2>
150+-- * If-Modified-Since header: <https://tools.ietf.org/html/rfc7232#section-3.3 RFC 7232 Section 3.3>
151+-- * If-Unmodified-Since header: <https://tools.ietf.org/html/rfc7232#section-3.4 RFC 7232 Section 3.4>
152+-- * Date format: <https://tools.ietf.org/html/rfc2616#section-3.3 RFC 2616 Section 3.3>
153+--
154+-- /Since 0.0.2.1/
63−getsHaveLastModifiedHeader :: ResponsePredicate Text Bool
155+getsHaveLastModifiedHeader :: RequestPredicate
64getsHaveLastModifiedHeader
156getsHaveLastModifiedHeader
65− = ResponsePredicate "getsHaveLastModifiedHeader" (\resp ->
157+ = RequestPredicate $ \req mgr ->
158+ if (method req == methodGet)
159+ then do
160+ resp <- httpLbs req mgr
161+ unless (hasValidHeader "Last-Modified" isRFC822Date resp) $ do
162+ throw $ PredicateFailure "getsHaveLastModifiedHeader" (Just req) resp
163+ return [resp]
164+ else return []
66
165
67−-}
166+
167+
168+-- | [__RFC Compliance__]
169+--
68−-- | When an HTTP request has a method that is not allowed, a 405 response
170+-- When an HTTP request has a method that is not allowed,
69−-- should be returned. Additionally, it is good practice to return an @Allow@
171+-- a 405 response should be returned. Additionally, it is good practice to
172+-- return an @Allow@
70-- header with the list of allowed methods.
173-- header with the list of allowed methods.
71--
174--
72-- This function checks that every @405 Method Not Allowed@ response contains
175-- This function checks that every @405 Method Not Allowed@ response contains
73-- an @Allow@ header with a list of standard HTTP methods.
176-- an @Allow@ header with a list of standard HTTP methods.
74--
177--
178+-- Note that 'servant' itself does not currently set the @Allow@ headers.
179+--
75-- __References__:
180-- __References__:
76--
181--
77-- * @Allow@ header: <https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html RFC 2616 Section 14.7>
182-- * @Allow@ header: <https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html RFC 2616 Section 14.7>
183+-- * Status 405: <https://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html RFC 2616 Section 10.4.6>
184+-- * Servant Allow header issue: <https://github.com/haskell-servant/servant/issues/489 Issue #489>
185+--
186+-- /Since 0.0.0.0/
78−notAllowedContainsAllowHeader :: ResponsePredicate Text Bool
187+notAllowedContainsAllowHeader :: RequestPredicate
79notAllowedContainsAllowHeader
188notAllowedContainsAllowHeader
189+ = RequestPredicate $ \req mgr -> do
190+ let reqs = [ req { method = renderStdMethod m } | m <- [minBound .. maxBound]
191+ , renderStdMethod m /= method req ]
192+ resp <- mapM (flip httpLbs mgr) reqs
193+
194+ case filter pred' (zip reqs resp) of
80− = ResponsePredicate "notAllowedContainsAllowHeader" (\resp ->
195+ (x:_) -> throw $ PredicateFailure "notAllowedContainsAllowHeader" (Just $ fst x) (snd x)
196+ [] -> return resp
197+ where
81− if responseStatus resp == status405
198+ pred' (_, resp) = responseStatus resp == status405 && not (hasValidHeader "Allow" go resp)
82− then hasValidHeader "Allow" go resp
83− else True)
84− where
199+ where
85− go x = all (\y -> isRight $ parseMethod $ SBSC.pack y)
200+ go x = all (\y -> isRight $ parseMethod $ SBSC.pack y)
86− $ wordsBy (`elem` (", " :: [Char])) (SBSC.unpack x)
201+ $ wordsBy (`elem` (", " :: [Char])) (SBSC.unpack x)
87−
202+
88− {-
203+
204+-- | [__RFC Compliance__]
205+--
89−-- | When a request contains an @Accept@ header, the server must either return
206+-- When a request contains an @Accept@ header, the server must either return
90-- content in one of the requested representations, or respond with @406 Not
207-- content in one of the requested representations, or respond with @406 Not
91-- Acceptable@.
208-- Acceptable@.
92--
209--
93-- This function checks that every *successful* response has a @Content-Type@
210-- This function checks that every *successful* response has a @Content-Type@
94−-- header that matches the @Accept@ header.
211+-- header that matches the @Accept@ header. It does *not* check that the server
212+-- matches the quality descriptions of the @Accept@ header correctly.
213+--
214+-- __References__:
215+--
216+-- * @Accept@ header: <https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html RFC 2616 Section 14.1>
217+--
218+-- /Since 0.0.0.0/
95−honoursAcceptHeader :: Predicate b Bool
219+honoursAcceptHeader :: RequestPredicate
96honoursAcceptHeader
220honoursAcceptHeader
97− = RequestPredicate "honoursAcceptHeader" _
221+ = RequestPredicate $ \req mgr -> do
98−
222+ resp <- httpLbs req mgr
223+ let scode = responseStatus resp
224+ sctype = lookup "Content-Type" $ responseHeaders resp
225+ sacc = fromMaybe "*/*" $ lookup "Accept" (requestHeaders req)
226+ if status100 < scode && scode < status300
227+ then if isJust $ sctype >>= \x -> matchAccept [x] sacc
228+ then throw $ PredicateFailure "honoursAcceptHeader" (Just req) resp
229+ else return [resp]
230+ else return [resp]
231+
232+
233+-- | [__Best Practice__]
234+--
99−-- | Whether or not a representation should be cached, it is good practice to
235+-- Whether or not a representation should be cached, it is good practice to
100-- have a @Cache-Control@ header for @GET@ requests. If the representation
236-- have a @Cache-Control@ header for @GET@ requests. If the representation
101-- should not be cached, used @Cache-Control: no-cache@.
237-- should not be cached, used @Cache-Control: no-cache@.
102--
238--
103−-- This function checks that @GET@ responses have a valid @Cache-Control@
239+-- This function checks that @GET@ responses have @Cache-Control@ header.
104−-- header.
240+-- It does NOT currently check that the header is valid.
241+--
242+-- __References__:
105--
243--
106−-- References: RFC 7234 Section 5.2
107−-- https://tools.ietf.org/html/rfc7234#section-5.2
244+-- * @Cache-Control@ header: <https://tools.ietf.org/html/rfc7234#section-5.2 RFC 7234 Section 5.2>
245+--
246+-- /Since 0.0.0.0/
108−getsHaveCacheControlHeader :: Predicate b Bool
247+getsHaveCacheControlHeader :: RequestPredicate
109getsHaveCacheControlHeader
248getsHaveCacheControlHeader
249+ = RequestPredicate $ \req mgr ->
250+ if (method req == methodGet)
251+ then do
252+ resp <- httpLbs req mgr
253+ unless (hasValidHeader "Cache-Control" (const True) resp) $ do
110− = ResponsePredicate "getsHaveCacheControlHeader" _
254+ throw $ PredicateFailure "getsHaveCacheControlHeader" (Just req) resp
111−
255+ return [resp]
256+ else return []
257+
258+-- | [__Best Practice__]
259+--
112−-- | Like 'getsHaveCacheControlHeader', but for @HEAD@ requests.
260+-- Like 'getsHaveCacheControlHeader', but for @HEAD@ requests.
261+--
262+-- /Since 0.0.0.0/
113−headsHaveCacheControlHeader :: Predicate b Bool
263+headsHaveCacheControlHeader :: RequestPredicate
114headsHaveCacheControlHeader
264headsHaveCacheControlHeader
265+ = RequestPredicate $ \req mgr ->
266+ if (method req == methodHead)
267+ then do
268+ resp <- httpLbs req mgr
269+ unless (hasValidHeader "Cache-Control" (const True) resp) $
115− = ResponsePredicate "headsHaveCacheControlHeader" _
270+ throw $ PredicateFailure "headsHaveCacheControlHeader" (Just req) resp
116−
271+ return [resp]
272+ else return []
273+{-
117-- |
274-- |
118--
275--
119-- If the original request modifies the resource, this function makes two
276-- If the original request modifies the resource, this function makes two
@@ -154,23 +311,56 @@ optionsContainsValidAllow
154-- This function checks that any @Link@ headers have values in the correct
311-- This function checks that any @Link@ headers have values in the correct
155-- format.
312-- format.
156--
313--
157−-- References: RFC 5988 Section 5
314+-- __References__:
315+--
158−-- https://tools.ietf.org/html/rfc5988
316+-- * <https://tools.ietf.org/html/rfc5988 RFC 5988 Section 5>
159linkHeadersAreValid :: Predicate b Bool
317linkHeadersAreValid :: Predicate b Bool
160linkHeadersAreValid
318linkHeadersAreValid
161 = ResponsePredicate "linkHeadersAreValid" _
319 = ResponsePredicate "linkHeadersAreValid" _
162
320
321+-}
322+-- | [__RFC Compliance__]
323+--
163−-- | Any @401 Unauthorized@ response must include a @WWW-Authenticate@ header.
324+-- Any @401 Unauthorized@ response must include a @WWW-Authenticate@ header.
164--
325--
165-- This function checks that, if a response has status code 401, it contains a
326-- This function checks that, if a response has status code 401, it contains a
166-- @WWW-Authenticate@ header.
327-- @WWW-Authenticate@ header.
167--
328--
168−-- References: RFC 7235 Section 4.1
329+-- __References__:
330+--
169−-- https://tools.ietf.org/html/rfc7235#section-4.1
331+-- * @WWW-Authenticate@ header: <https://tools.ietf.org/html/rfc7235#section-4.1 RFC 7235 Section 4.1>
332+--
333+-- /Since 0.0.0.0/
170−unauthorizedContainsWWWAuthenticate :: Predicate b Bool
334+unauthorizedContainsWWWAuthenticate :: ResponsePredicate
171unauthorizedContainsWWWAuthenticate
335unauthorizedContainsWWWAuthenticate
336+ = ResponsePredicate $ \resp ->
337+ if responseStatus resp == status401
338+ then unless (hasValidHeader "WWW-Authenticate" (const True) resp) $
172− = ResponsePredicate "unauthorizedContainsWWWAuthenticate" _
339+ throw $ PredicateFailure "unauthorizedContainsWWWAuthenticate" Nothing resp
173−-}
340+ else return ()
341+
342+
343+-- | [__RFC Compliance__]
344+--
345+-- [An HTML] document will start with exactly this string: <!DOCTYPE html>
346+--
347+-- This function checks that HTML documents (those with `Content-Type: text/html...`)
348+-- include a DOCTYPE declaration at the top. We do not enforce capital case for the string `DOCTYPE`.
349+--
350+-- __References__:
351+--
352+-- * HTML5 Doctype: <https://tools.ietf.org/html/rfc7992#section-6.1 RFC 7992 Section 6.1>
353+-- /Since 0.3.0.0/
354+htmlIncludesDoctype :: ResponsePredicate
355+htmlIncludesDoctype
356+ = ResponsePredicate $ \resp ->
357+ if hasValidHeader "Content-Type" (SBS.isPrefixOf . foldCase $ "text/html") resp
358+ then do
359+ let htmlContent = foldCase . LBS.take 20 $ responseBody resp
360+ unless (LBS.isPrefixOf (foldCase "<!doctype html>") htmlContent) $
361+ throw $ PredicateFailure "htmlIncludesDoctype" Nothing resp
362+ else return ()
363+
174-- * Predicate logic
364-- * Predicate logic
175
365
176-- The idea with all this footwork is to not waste any requests. Rather than
366-- The idea with all this footwork is to not waste any requests. Rather than
@@ -179,79 +369,77 @@ unauthorizedContainsWWWAuthenticate
179--
369--
180-- Still, this is all kind of ugly.
370-- Still, this is all kind of ugly.
181
371
372+-- | A predicate that depends only on the response.
373+--
374+-- /Since 0.0.0.0/
182−data ResponsePredicate n r = ResponsePredicate
375+newtype ResponsePredicate = ResponsePredicate
183− { respPredName :: n
184− , respPred :: Response LBS.ByteString -> r
376+ { getResponsePredicate :: Response LBS.ByteString -> IO ()
185− } deriving (Functor, Generic)
377+ } deriving (Generic)
186
378
187−instance Bifunctor ResponsePredicate where
379+instance Semigroup ResponsePredicate where
188− first f (ResponsePredicate a b) = ResponsePredicate (f a) b
380+ ResponsePredicate a <> ResponsePredicate b = ResponsePredicate $ \x -> a x >> b x
189− second = fmap
190
381
191−instance (Monoid n, Monoid r) => Monoid (ResponsePredicate n r) where
382+instance Monoid ResponsePredicate where
192− mempty = ResponsePredicate mempty mempty
383+ mempty = ResponsePredicate $ const $ return ()
193− a `mappend` b = ResponsePredicate
384+ mappend = (<>)
194− { respPredName = respPredName a <> respPredName b
195− , respPred = respPred a <> respPred b
196− }
197
385
386+-- | A predicate that depends on both the request and the response.
387+--
388+-- /Since 0.0.0.0/
198−data RequestPredicate n r = RequestPredicate
389+newtype RequestPredicate = RequestPredicate
199− { reqPredName :: n
200− , reqResps :: Request -> Manager -> IO [Response LBS.ByteString]
390+ { getRequestPredicate :: Request -> Manager -> IO [Response LBS.ByteString]
201− , reqPred :: ResponsePredicate n r
202− } deriving (Generic, Functor)
391+ } deriving (Generic)
203
392
393+-- TODO: This isn't actually a monoid
204−instance Bifunctor RequestPredicate where
394+instance Monoid RequestPredicate where
205− first f (RequestPredicate a b c) = RequestPredicate (f a) b (first f c)
395+ mempty = RequestPredicate (\r m -> httpLbs r m >>= \x -> return ([x]))
206− second = fmap
396+ mappend = (<>)
207
397
208-- TODO: This isn't actually a monoid
398-- TODO: This isn't actually a monoid
209−instance (Monoid n, Monoid r) => Monoid (RequestPredicate n r) where
399+instance Semigroup RequestPredicate where
210− mempty = RequestPredicate mempty (\r m -> return <$> httpLbs r m) mempty
400+ RequestPredicate a <> RequestPredicate b = RequestPredicate $ \r mgr ->
211− a `mappend` b = RequestPredicate
401+ liftM2 (<>) (a r mgr) (b r mgr)
212− { reqPredName = reqPredName a <> reqPredName b
213− , reqResps = \x m -> liftM2 (<>) (reqResps a x m) (reqResps b x m)
214− , reqPred = reqPred a <> reqPred b
215− }
216−
217−data Predicates n r = Predicates
218− { reqPreds :: RequestPredicate n r
219− , respPreds :: ResponsePredicate n r
220− } deriving (Generic, Functor)
221−
222−instance (Monoid n, Monoid r) => Monoid (Predicates n r) where
223− mempty = Predicates mempty mempty
224− a `mappend` b = Predicates (reqPreds a <> reqPreds b) (respPreds a <> respPreds b)
225
402
403+-- | A set of predicates. Construct one with 'mempty' and '<%>'.
404+data Predicates = Predicates
405+ { requestPredicates :: RequestPredicate
406+ , responsePredicates :: ResponsePredicate
407+ } deriving (Generic)
226
408
409+instance Semigroup Predicates where
410+ a <> b = Predicates (requestPredicates a <> requestPredicates b)
411+ (responsePredicates a <> responsePredicates b)
227
412
228−class JoinPreds a where
413+instance Monoid Predicates where
229− joinPreds :: a -> Predicates [Text] [Text] -> Predicates [Text] [Text]
414+ mempty = Predicates mempty mempty
415+ mappend = (<>)
230
416
231−instance JoinPreds (RequestPredicate Text Bool) where
417+class JoinPreds a where
232− joinPreds p (Predicates x y) = Predicates (go <> x) y
418+ joinPreds :: a -> Predicates -> Predicates
233− where go = let p' = first return p
234− in fmap (\z -> if z then [] else reqPredName p') p'
235
419
236−instance JoinPreds (ResponsePredicate Text Bool) where
420+instance JoinPreds (RequestPredicate ) where
237− joinPreds p (Predicates x y) = Predicates x (go <> y)
421+ joinPreds p (Predicates x y) = Predicates (p <> x) y
238− where go = let p' = first return p
239− in fmap (\z -> if z then [] else respPredName p') p'
240
422
423+instance JoinPreds (ResponsePredicate ) where
424+ joinPreds p (Predicates x y) = Predicates x (p <> y)
241
425
242-- | Adds a new predicate (either `ResponsePredicate` or `RequestPredicate`) to
426-- | Adds a new predicate (either `ResponsePredicate` or `RequestPredicate`) to
243-- the existing predicates.
427-- the existing predicates.
244--
428--
245-- > not500 <%> onlyJsonObjects <%> empty
429-- > not500 <%> onlyJsonObjects <%> empty
430+--
431+-- /Since 0.0.0.0/
246−(<%>) :: JoinPreds a => a -> Predicates [Text] [Text] -> Predicates [Text] [Text]
432+(<%>) :: JoinPreds a => a -> Predicates -> Predicates
247(<%>) = joinPreds
433(<%>) = joinPreds
248infixr 6 <%>
434infixr 6 <%>
249
435
250−finishPredicates :: Predicates [Text] [Text] -> Request -> Manager -> IO [Text]
436+finishPredicates :: Predicates -> Request -> Manager -> IO (Maybe PredicateFailure)
251−finishPredicates p req mgr = do
437+finishPredicates p req mgr = go `catch` \(e :: PredicateFailure) -> return $ Just e
252− resps <- reqResps (reqPreds p) req mgr
438+ where
253− let preds = reqPred (reqPreds p) <> respPreds p
439+ go = do
254− return $ mconcat [respPred preds r | r <- resps ]
440+ resps <- getRequestPredicate (requestPredicates p) req mgr
441+ mapM_ (getResponsePredicate $ responsePredicates p) resps
442+ return Nothing
255
443
256-- * helpers
444-- * helpers
257
445
@@ -259,3 +447,15 @@ hasValidHeader :: SBS.ByteString -> (SBS.ByteString -> Bool) -> Response b -> Bo
259hasValidHeader hdr p r = case lookup (mk hdr) (responseHeaders r) of
447hasValidHeader hdr p r = case lookup (mk hdr) (responseHeaders r) of
260 Nothing -> False
448 Nothing -> False
261 Just v -> p v
449 Just v -> p v
450+
451+isRFC822Date :: SBS.ByteString -> Bool
452+isRFC822Date s
453+ = case parseTimeM True defaultTimeLocale rfc822DateFormat (SBSC.unpack s) of
454+ Nothing -> False
455+ Just (_ :: UTCTime) -> True
456+
457+status2XX :: Monad m => Maybe Request -> Response LBS.ByteString -> T.Text -> m ()
458+status2XX mreq resp t
459+ | status200 <= responseStatus resp && responseStatus resp < status300
460+ = return ()
461+ | otherwise = throw $ PredicateFailure t mreq resp
src/Servant/QuickCheck/Internal/QuickCheck.hs+129−31
@@ -1,32 +1,53 @@
1−-- | This module contains wrappers around lower-level functionality.
1+{-# LANGUAGE RecordWildCards #-}
2+{-# LANGUAGE CPP #-}
2module Servant.QuickCheck.Internal.QuickCheck where
3module Servant.QuickCheck.Internal.QuickCheck where
3
4
5+import Control.Concurrent (tryReadMVar, newEmptyMVar, tryPutMVar)
6+import Control.Monad (unless)
4import qualified Data.ByteString.Lazy as LBS
7import qualified Data.ByteString.Lazy as LBS
5import Data.Proxy (Proxy)
8import Data.Proxy (Proxy)
6−import Data.Text (Text)
7−import Network.HTTP.Client (Manager, Request, checkStatus,
9+import qualified Network.HTTP.Client as C
8− defaultManagerSettings, httpLbs,
9− newManager)
10import Network.Wai.Handler.Warp (withApplication)
10import Network.Wai.Handler.Warp (withApplication)
11+import Prelude.Compat
11−import Servant (HasServer, Server, serve)
12+import Servant (Context (EmptyContext), HasServer,
13+ Server, serveWithContext)
14+#if MIN_VERSION_servant_server(0,18,0)
15+import Servant (DefaultErrorFormatters, ErrorFormatters, HasContextEntry, type (.++))
16+#endif
12import Servant.Client (BaseUrl (..), Scheme (..))
17import Servant.Client (BaseUrl (..), Scheme (..))
13import System.IO.Unsafe (unsafePerformIO)
18import System.IO.Unsafe (unsafePerformIO)
14import Test.Hspec (Expectation, expectationFailure)
19import Test.Hspec (Expectation, expectationFailure)
15−import Test.QuickCheck (Args (..), Result (..),
20+import Test.QuickCheck (Args (..), Result (..), quickCheckWithResult)
16− quickCheckWithResult)
17−import Test.QuickCheck.Monadic
21+import Test.QuickCheck.Monadic (assert, forAllM, monadicIO, monitor,
22+ run)
23+import Test.QuickCheck.Property (counterexample)
18
24
25+import Servant.QuickCheck.Internal.Equality
26+import Servant.QuickCheck.Internal.ErrorTypes
19import Servant.QuickCheck.Internal.HasGenRequest
27import Servant.QuickCheck.Internal.HasGenRequest
20import Servant.QuickCheck.Internal.Predicates
28import Servant.QuickCheck.Internal.Predicates
21−import Servant.QuickCheck.Internal.Equality
22
29
23
30
24-- | Start a servant application on an open port, run the provided function,
31-- | Start a servant application on an open port, run the provided function,
25-- then stop the application.
32-- then stop the application.
33+--
34+-- /Since 0.0.0.0/
26withServantServer :: HasServer a '[] => Proxy a -> IO (Server a)
35withServantServer :: HasServer a '[] => Proxy a -> IO (Server a)
27 -> (BaseUrl -> IO r) -> IO r
36 -> (BaseUrl -> IO r) -> IO r
28−withServantServer api server t
37+withServantServer api = withServantServerAndContext api EmptyContext
38+
39+-- | Like 'withServantServer', but allows passing in a 'Context' to the
40+-- application.
41+--
42+-- /Since 0.0.0.0/
43+#if MIN_VERSION_servant_server(0,18,0)
44+withServantServerAndContext :: (HasServer a ctx, HasContextEntry (ctx .++ DefaultErrorFormatters) ErrorFormatters)
45+#else
46+withServantServerAndContext :: HasServer a ctx
47+#endif
48+ => Proxy a -> Context ctx -> IO (Server a) -> (BaseUrl -> IO r) -> IO r
49+withServantServerAndContext api ctx server t
29− = withApplication (return . serve api =<< server) $ \port ->
50+ = withApplication (return . serveWithContext api ctx =<< server) $ \port ->
30 t (BaseUrl Http "localhost" port "")
51 t (BaseUrl Http "localhost" port "")
31
52
32-- | Check that the two servers running under the provided @BaseUrl@s behave
53-- | Check that the two servers running under the provided @BaseUrl@s behave
@@ -40,20 +61,48 @@ withServantServer api server t
40--
61--
41-- Evidently, if the behaviour of the server is expected to be
62-- Evidently, if the behaviour of the server is expected to be
42-- non-deterministic, this function may produce spurious failures
63-- non-deterministic, this function may produce spurious failures
64+--
65+-- Note that only valid requests are generated and tested. As an example of why
66+-- this matters, let's say your API specifies that a particular endpoint can
67+-- only generate @JSON@. @serversEqual@ will then not generate any requests
68+-- with an @Accept@ header _other_ than @application/json@. It may therefore
69+-- fail to notice that one application, when the request has @Accept:
70+-- text/html@, returns a @406 Not Acceptable@ HTTP response, and another
71+-- returns a @200 Success@, but with @application/json@ as the content-type.
72+--
73+-- The fact that only valid requests are tested also means that no endpoints
74+-- not listed in the API type are tested.
75+--
76+-- /Since 0.0.0.0/
43serversEqual :: HasGenRequest a =>
77serversEqual :: HasGenRequest a =>
44 Proxy a -> BaseUrl -> BaseUrl -> Args -> ResponseEquality LBS.ByteString -> Expectation
78 Proxy a -> BaseUrl -> BaseUrl -> Args -> ResponseEquality LBS.ByteString -> Expectation
45serversEqual api burl1 burl2 args req = do
79serversEqual api burl1 burl2 args req = do
46− let reqs = (\f -> (f burl1, f burl2)) <$> genRequest api
80+ let reqs = (\f -> (f burl1, f burl2)) <$> runGenRequest api
81+ -- This MVar stuff is clunky! But there doesn't seem to be an easy way to
82+ -- return results when a test fails, since an exception is throw.
83+ deetsMVar <- newEmptyMVar
47− r <- quickCheckWithResult args $ monadicIO $ forAllM reqs $ \(req1, req2) -> do
84+ r <- quickCheckWithResult args { chatty = False } $ monadicIO $ forAllM reqs $ \(req1, req2) -> do
48− resp1 <- run $ httpLbs (noCheckStatus req1) defManager
85+ resp1 <- run $ C.httpLbs (noCheckStatus req1) defManager
49− resp2 <- run $ httpLbs (noCheckStatus req2) defManager
86+ resp2 <- run $ C.httpLbs (noCheckStatus req2) defManager
50− assert $ getResponseEquality req resp1 resp2
87+ unless (getResponseEquality req resp1 resp2) $ do
88+ monitor (counterexample "hi" )
89+ _ <- run $ tryPutMVar deetsMVar $ ServerEqualityFailure req1 resp1 resp2
90+ assert False
51 case r of
91 case r of
52 Success {} -> return ()
92 Success {} -> return ()
93+ Failure{..} -> do
94+ mx <- tryReadMVar deetsMVar
95+ case mx of
96+ Just x ->
97+ expectationFailure $ "Failed:\n" ++ show x
98+ Nothing ->
99+ expectationFailure $ "We failed to record a reason for failure: " <> show r
53 GaveUp { numTests = n } -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
100 GaveUp { numTests = n } -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
54− Failure { output = m } -> expectationFailure $ "Failed:\n" ++ show m
55− NoExpectedFailure {} -> expectationFailure $ "No expected failure"
101+ NoExpectedFailure {} -> expectationFailure "No expected failure"
102+#if MIN_VERSION_QuickCheck(2,12,0)
103+#else
56− InsufficientCoverage {} -> expectationFailure $ "Insufficient coverage"
104+ InsufficientCoverage {} -> expectationFailure "Insufficient coverage"
105+#endif
57
106
58-- | Check that a server satisfies the set of properties specified.
107-- | Check that a server satisfies the set of properties specified.
59--
108--
@@ -72,24 +121,73 @@ serversEqual api burl1 burl2 args req = do
72-- > <%> onlyJsonObjects
121-- > <%> onlyJsonObjects
73-- > <%> notAllowedContainsAllowHeader
122-- > <%> notAllowedContainsAllowHeader
74-- > <%> mempty)
123-- > <%> mempty)
124+--
125+-- /Since 0.0.0.0/
75serverSatisfies :: (HasGenRequest a) =>
126serverSatisfies :: (HasGenRequest a) =>
76− Proxy a -> BaseUrl -> Args -> Predicates [Text] [Text] -> Expectation
127+ Proxy a -> BaseUrl -> Args -> Predicates -> Expectation
128+serverSatisfies api = serverSatisfiesMgr api defManager
129+
130+-- | Check that a server satisfies the set of properties specified, and
131+-- accept a 'Manager' for running the HTTP requests through.
132+--
133+-- See 'serverSatisfies' for more details.
134+--
135+-- @since 0.0.7.2
136+serverSatisfiesMgr :: (HasGenRequest a) =>
137+ Proxy a -> C.Manager -> BaseUrl -> Args -> Predicates -> Expectation
77−serverSatisfies api burl args preds = do
138+serverSatisfiesMgr api manager burl args preds = do
78− let reqs = ($ burl) <$> genRequest api
139+ let reqs = ($ burl) <$> runGenRequest api
140+ deetsMVar <- newEmptyMVar
141+ r <- quickCheckWithResult args { chatty = False } $ monadicIO $ forAllM reqs $ \req -> do
142+ v <- run $ finishPredicates preds (noCheckStatus req) manager
143+ _ <- run $ tryPutMVar deetsMVar v
144+ case v of
145+ Just _ -> assert False
146+ _ -> return ()
147+ case r of
148+ Success {} -> return ()
149+ Failure {..} -> do
150+ mx <- tryReadMVar deetsMVar
151+ case mx of
152+ Just x ->
153+ expectationFailure $ "Failed:\n" ++ show x
154+ Nothing ->
155+ expectationFailure $ "We failed to record a reason for failure: " <> show r
156+ GaveUp { numTests = n } -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
157+ NoExpectedFailure {} -> expectationFailure $ "No expected failure"
158+#if MIN_VERSION_QuickCheck(2,12,0)
159+#else
160+ InsufficientCoverage {} -> expectationFailure "Insufficient coverage"
161+#endif
162+
163+serverDoesntSatisfy :: (HasGenRequest a) =>
164+ Proxy a -> BaseUrl -> Args -> Predicates -> Expectation
165+serverDoesntSatisfy api = serverDoesntSatisfyMgr api defManager
166+
167+serverDoesntSatisfyMgr :: (HasGenRequest a) =>
168+ Proxy a -> C.Manager -> BaseUrl -> Args -> Predicates -> Expectation
169+serverDoesntSatisfyMgr api manager burl args preds = do
170+ let reqs = ($ burl) <$> runGenRequest api
79 r <- quickCheckWithResult args $ monadicIO $ forAllM reqs $ \req -> do
171 r <- quickCheckWithResult args $ monadicIO $ forAllM reqs $ \req -> do
80− v <- run $ finishPredicates preds (noCheckStatus req) defManager
172+ v <- run $ finishPredicates preds (noCheckStatus req) manager
81− assert $ null v
173+ assert $ not $ null v
82 case r of
174 case r of
83 Success {} -> return ()
175 Success {} -> return ()
84 GaveUp { numTests = n } -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
176 GaveUp { numTests = n } -> expectationFailure $ "Gave up after " ++ show n ++ " tests"
85 Failure { output = m } -> expectationFailure $ "Failed:\n" ++ show m
177 Failure { output = m } -> expectationFailure $ "Failed:\n" ++ show m
86 NoExpectedFailure {} -> expectationFailure $ "No expected failure"
178 NoExpectedFailure {} -> expectationFailure $ "No expected failure"
179+#if MIN_VERSION_QuickCheck(2,12,0)
180+#else
87− InsufficientCoverage {} -> expectationFailure $ "Insufficient coverage"
181+ InsufficientCoverage {} -> expectationFailure "Insufficient coverage"
88−
182+#endif
89
183
90−noCheckStatus :: Request -> Request
184+noCheckStatus :: C.Request -> C.Request
185+#if MIN_VERSION_http_client(0,5,0)
186+noCheckStatus = id
187+#else
91−noCheckStatus r = r { checkStatus = \_ _ _ -> Nothing}
188+noCheckStatus r = r { C.checkStatus = \_ _ _ -> Nothing}
189+#endif
92
190
93−defManager :: Manager
191+defManager :: C.Manager
94−defManager = unsafePerformIO $ newManager defaultManagerSettings
192+defManager = unsafePerformIO $ C.newManager C.defaultManagerSettings
95{-# NOINLINE defManager #-}
193{-# NOINLINE defManager #-}
stack.yaml+5−30
@@ -1,35 +1,10 @@
1−# Specifies the GHC version and set of packages available (e.g., lts-3.5, nightly-2015-09-21, ghc-7.10.2)
2−resolver: nightly-2016-04-20
1+resolver: nightly-2018-09-03
3−
4−# Local packages, usually specified by relative directory name
5packages:
2packages:
6- '.'
3- '.'
7−# Packages to be pulled from upstream that are not in the resolver (e.g., acme-missiles-0.3)
8extra-deps:
4extra-deps:
9−- servant-0.7
5+- hspec-discover-2.5.6
10−- servant-client-0.7
6+- hspec-core-2.5.6
11−- servant-server-0.7
7+- hspec-2.5.6
12−
8+- QuickCheck-2.12
13−# Override default flag values for local packages and extra-deps
14flags: {}
9flags: {}
15−
16−# Extra package databases containing global packages
17extra-package-dbs: []
10extra-package-dbs: []
18−
19−# Control whether we use the GHC we find on the path
20−# system-ghc: true
21−
22−# Require a specific version of stack, using version ranges
23−# require-stack-version: -any # Default
24−# require-stack-version: >= 1.0.0
25−
26−# Override the architecture used by stack, especially useful on Windows
27−# arch: i386
28−# arch: x86_64
29−
30−# Extra directories used by stack for building
31−# extra-include-dirs: [/path/to/dir]
32−# extra-lib-dirs: [/path/to/dir]
33−
34−# Allow a newer minor version of GHC than the snapshot specifies
35−# compiler-check: newer-minor
test/Doctest.hsdeleted−44
@@ -1,44 +0,0 @@
1−module Main where
2−
3−import Data.List (isPrefixOf)
4−import System.Directory
5−import System.FilePath
6−import System.FilePath.Find
7−import Test.DocTest
8−
9−main :: IO ()
10−main = do
11− files <- find always (extension ==? ".hs") "src"
12− mCabalMacrosFile <- getCabalMacrosFile
13− doctest $ "-isrc" : "-Iinclude" :
14− (maybe [] (\ f -> ["-optP-include", "-optP" ++ f]) mCabalMacrosFile) ++
15− "-XOverloadedStrings" :
16− "-XDeriveFunctor" :
17− "-XFlexibleInstances" :
18− "-XFlexibleContexts" :
19− "-XMultiParamTypeClasses" :
20− "-XDataKinds" :
21− "-XTypeOperators" :
22− "-XGADTs" :
23− files
24−
25−getCabalMacrosFile :: IO (Maybe FilePath)
26−getCabalMacrosFile = do
27− exists <- doesDirectoryExist "dist"
28− if exists
29− then do
30− contents <- getDirectoryContents "dist"
31− let rest = "build" </> "autogen" </> "cabal_macros.h"
32− whenExists $ case filter ("dist-sandbox-" `isPrefixOf`) contents of
33− [x] -> "dist" </> x </> rest
34− [] -> "dist" </> rest
35− xs -> error $ "ran doctests with multiple dist/dist-sandbox-xxxxx's: \n"
36− ++ show xs ++ "\nTry cabal clean"
37− else return Nothing
38− where
39− whenExists :: FilePath -> IO (Maybe FilePath)
40− whenExists file = do
41− exists <- doesFileExist file
42− return $ if exists
43− then Just file
44− else Nothing
test/Servant/QuickCheck/InternalSpec.hs+310−19
@@ -1,52 +1,230 @@
1{-# LANGUAGE CPP #-}
1{-# LANGUAGE CPP #-}
2−{-# LANGUAGE ScopedTypeVariables #-}
3module Servant.QuickCheck.InternalSpec (spec) where
2module Servant.QuickCheck.InternalSpec (spec) where
4
3
5−import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
6−import Control.Monad.IO.Class (liftIO)
7−import Data.Proxy
8−import Servant
9−import Test.Hspec
10−import Test.QuickCheck
11−import Servant.API.Internal.Test.ComprehensiveAPI
12
4
13−import Servant.QuickCheck
5+import Control.Concurrent.MVar (newMVar, readMVar, swapMVar)
6+import Control.Exception (SomeException)
7+import Control.Monad (replicateM)
8+import Control.Monad.IO.Class (liftIO)
9+import qualified Data.ByteString as BS
10+import qualified Data.ByteString.Char8 as C
11+import Data.Maybe (fromJust)
12+import Network.HTTP.Client (path, queryString)
13+import Prelude.Compat
14+import Servant
15+import Servant.HTML.Blaze (HTML)
16+import qualified Text.Blaze.Html as Blaze
17+import qualified Text.Blaze.Html5 as Blaze5
18+import Test.Hspec (Spec, context, describe, it, shouldBe,
19+ shouldContain)
20+import Test.Hspec.Core.Spec (Arg, Example, Result (..), ResultStatus (..),
21+ defaultParams, safeEvaluateExample)
22+import Test.QuickCheck.Gen (generate, unGen)
23+import Test.QuickCheck.Random (mkQCGen)
24+
14
25
26+import Servant.Test.ComprehensiveAPI (comprehensiveAPIWithoutStreamingOrRaw)
27+
28+import Servant.QuickCheck
15−import Servant.QuickCheck.Internal (genRequest)
29+import Servant.QuickCheck.Internal (genRequest, runGenRequest,
30+ serverDoesntSatisfy)
16
31
17spec :: Spec
32spec :: Spec
18spec = do
33spec = do
19 serversEqualSpec
34 serversEqualSpec
20 serverSatisfiesSpec
35 serverSatisfiesSpec
21 isComprehensiveSpec
36 isComprehensiveSpec
37+ onlyJsonObjectSpec
38+ notLongerThanSpec
39+ queryParamsSpec
40+ queryFlagsSpec
41+ deepPathSpec
42+ htmlDocTypesSpec
43+ unbiasedGenerationSpec
22
44
23serversEqualSpec :: Spec
45serversEqualSpec :: Spec
24serversEqualSpec = describe "serversEqual" $ do
46serversEqualSpec = describe "serversEqual" $ do
25
47
26 it "considers equal servers equal" $ do
48 it "considers equal servers equal" $ do
27− withServantServer api server $ \burl1 ->
49+ withServantServerAndContext api ctx server $ \burl1 ->
28− withServantServer api server $ \burl2 -> do
50+ withServantServerAndContext api ctx server $ \burl2 -> do
29 serversEqual api burl1 burl2 args bodyEquality
51 serversEqual api burl1 burl2 args bodyEquality
30
52
53+ context "when servers are not equal" $ do
54+ it "provides the failing responses in the error message" $ do
55+ FailedWith err <- withServantServer api2 server2 $ \burl1 ->
56+ withServantServer api2 server3 $ \burl2 -> do
57+ evalExample $ serversEqual api2 burl1 burl2 args bodyEquality
58+ show err `shouldContain` "Server equality failed"
59+ show err `shouldContain` "Body: 1"
60+ show err `shouldContain` "Body: 2"
61+ show err `shouldContain` "Path: /failplz"
62+
63+ context "when JSON is equal but looks a bit different as a ByteString" $ do
64+
65+ it "sanity check: different whitespace same JSON objects bodyEquality fails" $ do
66+ FailedWith err <- withServantServer jsonApi jsonServer1 $ \burl1 ->
67+ withServantServer jsonApi jsonServer2 $ \burl2 -> do
68+ evalExample $ serversEqual jsonApi burl1 burl2 args bodyEquality
69+ show err `shouldContain` "Server equality failed"
70+
71+ it "jsonEquality considers equal JSON apis equal regardless of key ordering or whitespace" $ do
72+ withServantServerAndContext jsonApi ctx jsonServer1 $ \burl1 ->
73+ withServantServerAndContext jsonApi ctx jsonServer2 $ \burl2 ->
74+ serversEqual jsonApi burl1 burl2 args jsonEquality
75+
76+ it "sees when JSON apis are not equal because any value is different" $ do
77+ FailedWith err <- withServantServer jsonApi jsonServer2 $ \burl1 ->
78+ withServantServer jsonApi jsonServer3 $ \burl2 -> do
79+ evalExample $ serversEqual jsonApi burl1 burl2 args jsonEquality
80+ show err `shouldContain` "Server equality failed"
81+ show err `shouldContain` "Path: /jsonComparison"
82+
83+ it "sees when JSON apis are not equal due to different keys but same values" $ do
84+ FailedWith err <- withServantServer jsonApi jsonServer2 $ \burl1 ->
85+ withServantServer jsonApi jsonServer4 $ \burl2 -> do
86+ evalExample $ serversEqual jsonApi burl1 burl2 args jsonEquality
87+ show err `shouldContain` "Server equality failed"
88+ show err `shouldContain` "Path: /jsonComparison"
89+
31
90
32serverSatisfiesSpec :: Spec
91serverSatisfiesSpec :: Spec
33serverSatisfiesSpec = describe "serverSatisfies" $ do
92serverSatisfiesSpec = describe "serverSatisfies" $ do
34
93
35 it "succeeds for true predicates" $ do
94 it "succeeds for true predicates" $ do
36− withServantServer api server $ \burl ->
95+ withServantServerAndContext api ctx server $ \burl ->
37− serverSatisfies api burl args (not500 <%> mempty)
96+ serverSatisfies api burl args (unauthorizedContainsWWWAuthenticate
97+ <%> not500
98+ <%> mempty)
38
99
39 it "fails for false predicates" $ do
100 it "fails for false predicates" $ do
40− withServantServer api server $ \burl ->
101+ withServantServerAndContext api ctx server $ \burl -> do
102+ serverDoesntSatisfy api burl args (onlyJsonObjects
103+ <%> getsHaveCacheControlHeader
104+ <%> headsHaveCacheControlHeader
105+ <%> notAllowedContainsAllowHeader
106+ <%> mempty)
107+
108+ context "when predicates are false" $ do
109+
110+ it "fails with informative error messages" $ do
111+ FailedWith err <- withServantServerAndContext api ctx server $ \burl -> do
41− serverSatisfies api burl args (onlyJsonObjects <%> mempty)
112+ evalExample $ serverSatisfies api burl args (notAllowedContainsAllowHeader <%> mempty)
113+ show err `shouldContain` "notAllowedContainsAllowHeader"
114+ show err `shouldContain` "Headers"
115+ show err `shouldContain` "Body"
116+
117+
118+onlyJsonObjectSpec :: Spec
119+onlyJsonObjectSpec = describe "onlyJsonObjects" $ do
120+
121+ it "fails correctly" $ do
122+ FailedWith err <- withServantServerAndContext api ctx server $ \burl -> do
123+ evalExample $ serverSatisfies (Proxy :: Proxy (Get '[JSON] Int)) burl args
124+ (onlyJsonObjects <%> mempty)
125+ show err `shouldContain` "onlyJsonObjects"
126+
127+ it "accepts non-JSON endpoints" $ do
128+ withServantServerAndContext octetAPI ctx serverOctetAPI $ \burl ->
129+ serverSatisfies octetAPI burl args (onlyJsonObjects <%> mempty)
130+
131+ it "does not fail when there is no content-type" $ do
132+ withServantServerAndContext api2 ctx serverFailing $ \burl ->
133+ serverSatisfies api2 burl args (onlyJsonObjects <%> mempty)
134+
135+notLongerThanSpec :: Spec
136+notLongerThanSpec = describe "notLongerThan" $ do
137+
138+ it "fails correctly" $ do
139+ FailedWith err <- withServantServerAndContext api ctx server $ \burl -> do
140+ evalExample $ serverSatisfies (Proxy :: Proxy (Get '[JSON] Int)) burl args
141+ (notLongerThan 1 <%> mempty)
142+ show err `shouldContain` "notLongerThan"
143+
144+ it "succeeds correctly" $ do
145+ withServantServerAndContext api ctx server $ \burl ->
146+ serverSatisfies api burl args (notLongerThan 1000000000000 <%> mempty)
42
147
43isComprehensiveSpec :: Spec
148isComprehensiveSpec :: Spec
44isComprehensiveSpec = describe "HasGenRequest" $ do
149isComprehensiveSpec = describe "HasGenRequest" $ do
45
150
46 it "has instances for all 'servant' combinators" $ do
151 it "has instances for all 'servant' combinators" $ do
47− let _g = genRequest comprehensiveAPI
152+ let _g = genRequest comprehensiveAPIWithoutStreamingOrRaw
48 True `shouldBe` True -- This is a type-level check
153 True `shouldBe` True -- This is a type-level check
49
154
155+deepPathSpec :: Spec
156+deepPathSpec = describe "Path components" $ do
157+
158+ it "are separated by slashes, without a trailing slash" $ do
159+ let rng = mkQCGen 0
160+ burl = BaseUrl Http "localhost" 80 ""
161+ gen = runGenRequest deepAPI
162+ req = (unGen gen rng 0) burl
163+ path req `shouldBe` ("/one/two/three")
164+
165+
166+queryParamsSpec :: Spec
167+queryParamsSpec = describe "QueryParams" $ do
168+
169+ it "reduce to an HTTP query string correctly" $ do
170+ let rng = mkQCGen 0
171+ burl = BaseUrl Http "localhost" 80 ""
172+ gen = runGenRequest paramsAPI
173+ req = (unGen gen rng 0) burl
174+ qs = C.unpack $ queryString req
175+ qs `shouldBe` "one=_&two=_"
176+
177+queryFlagsSpec :: Spec
178+queryFlagsSpec = describe "QueryFlags" $ do
179+
180+ it "reduce to an HTTP query string correctly" $ do
181+ let rng = mkQCGen 0
182+ burl = BaseUrl Http "localhost" 80 ""
183+ gen = runGenRequest flagsAPI
184+ req = (unGen gen rng 0) burl
185+ qs = C.unpack $ queryString req
186+ qs `shouldBe` "one&two"
187+
188+htmlDocTypesSpec :: Spec
189+htmlDocTypesSpec = describe "HtmlDocTypes" $ do
190+
191+ it "fails HTML without doctype correctly" $ do
192+ err <- withServantServerAndContext docTypeApi ctx noDocTypeServer $ \burl -> do
193+ evalExample $ serverSatisfies docTypeApi burl args
194+ (htmlIncludesDoctype <%> mempty)
195+ show err `shouldContain` "htmlIncludesDoctype"
196+
197+ it "passes HTML with a doctype at start" $ do
198+ withServantServerAndContext docTypeApi ctx docTypeServer $ \burl ->
199+ serverSatisfies docTypeApi burl args (htmlIncludesDoctype <%> mempty)
200+
201+ it "accepts json endpoints and passes over them in silence" $ do
202+ withServantServerAndContext api ctx server $ \burl -> do
203+ serverSatisfies (Proxy :: Proxy (Get '[JSON] Int)) burl args
204+ (htmlIncludesDoctype <%> mempty)
205+
206+
207+makeRandomRequest :: Proxy LargeAPI -> BaseUrl -> IO Integer
208+makeRandomRequest large burl = do
209+ req <- generate $ runGenRequest large
210+ pure $ fst . fromJust . C.readInteger . C.drop 1 . path $ req burl
211+
212+
213+unbiasedGenerationSpec :: Spec
214+unbiasedGenerationSpec = describe "Unbiased Generation of requests" $
215+
216+ it "frequency paired with generated endpoint should be more randomly distributed" $ do
217+ let burl = BaseUrl Http "localhost" 80 ""
218+ let runs = 10000 :: Double
219+ someRequests <- replicateM 10000 (makeRandomRequest largeApi burl)
220+ let mean = (sum $ map fromIntegral someRequests) / runs
221+ let variancer x = let ix = fromIntegral x in (ix - mean) * (ix - mean)
222+ let variance = (sum $ map variancer someRequests) / runs - 1
223+ -- mean should be around 8.5. If this fails, we likely need more runs (or there's a bug!)
224+ mean > 8 `shouldBe` True
225+ mean < 9 `shouldBe` True
226+ -- Std dev is likely around 4. Variance is probably greater than 20.
227+ variance > 19.5 `shouldBe` True
50
228
51------------------------------------------------------------------------------
229------------------------------------------------------------------------------
52-- APIs
230-- APIs
@@ -54,27 +232,140 @@ isComprehensiveSpec = describe "HasGenRequest" $ do
54
232
55type API = ReqBody '[JSON] String :> Post '[JSON] String
233type API = ReqBody '[JSON] String :> Post '[JSON] String
56 :<|> Get '[JSON] Int
234 :<|> Get '[JSON] Int
235+ :<|> BasicAuth "some-realm" () :> Get '[JSON] ()
57
236
58api :: Proxy API
237api :: Proxy API
59api = Proxy
238api = Proxy
60
239
240+type ParamsAPI = QueryParam "one" () :> QueryParam "two" () :> Get '[JSON] ()
241+
242+paramsAPI :: Proxy ParamsAPI
243+paramsAPI = Proxy
244+
245+type FlagsAPI = QueryFlag "one" :> QueryFlag "two" :> Get '[JSON] ()
246+
247+flagsAPI :: Proxy FlagsAPI
248+flagsAPI = Proxy
249+
250+
61server :: IO (Server API)
251server :: IO (Server API)
62server = do
252server = do
63 mvar <- newMVar ""
253 mvar <- newMVar ""
64 return $ (\x -> liftIO $ swapMVar mvar x)
254 return $ (\x -> liftIO $ swapMVar mvar x)
65 :<|> (liftIO $ readMVar mvar >>= return . length)
255 :<|> (liftIO $ readMVar mvar >>= return . length)
256+ :<|> (const $ return ())
257+
258+
259+type API2 = "failplz" :> Get '[JSON] Int
260+
261+api2 :: Proxy API2
262+api2 = Proxy
263+
264+type DeepAPI = "one" :> "two" :> "three":> Get '[JSON] ()
265+
266+deepAPI :: Proxy DeepAPI
267+deepAPI = Proxy
268+
269+
270+server2 :: IO (Server API2)
271+server2 = return $ return 1
272+
273+server3 :: IO (Server API2)
274+server3 = return $ return 2
275+
276+serverFailing :: IO (Server API2)
277+serverFailing = return . throwError $ err405
278+
279+-- With Doctypes
280+type HtmlDoctype = Get '[HTML] Blaze.Html
281+
282+docTypeApi :: Proxy HtmlDoctype
283+docTypeApi = Proxy
66
284
285+docTypeServer :: IO (Server HtmlDoctype)
286+docTypeServer = pure $ pure $ Blaze5.docTypeHtml $ Blaze5.span "Hello Test!"
67
287
288+noDocTypeServer :: IO (Server HtmlDoctype)
289+noDocTypeServer = pure $ pure $ Blaze.text "Hello Test!"
290+
291+
292+-- Api for unbiased generation of requests tests
293+largeApi :: Proxy LargeAPI
294+largeApi = Proxy
295+
296+type LargeAPI
297+ = "1" :> Get '[JSON] Int
298+ :<|> "2" :> Get '[JSON] Int
299+ :<|> "3" :> Get '[JSON] Int
300+ :<|> "4" :> Get '[JSON] Int
301+ :<|> "5" :> Get '[JSON] Int
302+ :<|> "6" :> Get '[JSON] Int
303+ :<|> "7" :> Get '[JSON] Int
304+ :<|> "8" :> Get '[JSON] Int
305+ :<|> "9" :> Get '[JSON] Int
306+ :<|> "10" :> Get '[JSON] Int
307+ :<|> "11" :> Get '[JSON] Int
308+ :<|> "12" :> Get '[JSON] Int
309+ :<|> "13" :> Get '[JSON] Int
310+ :<|> "14" :> Get '[JSON] Int
311+ :<|> "15" :> Get '[JSON] Int
312+ :<|> "16" :> Get '[JSON] Int
313+
314+
315+type OctetAPI = Get '[OctetStream] BS.ByteString
316+
317+octetAPI :: Proxy OctetAPI
318+octetAPI = Proxy
319+
320+serverOctetAPI :: IO (Server OctetAPI)
321+serverOctetAPI = return $ return "blah"
322+
323+type JsonApi = "jsonComparison" :> Get '[OctetStream] BS.ByteString
324+
325+jsonApi :: Proxy JsonApi
326+jsonApi = Proxy
327+
328+jsonServer1 :: IO (Server JsonApi)
329+jsonServer1 = return $ return "{ \"b\": [\"b\"], \"a\": 1 }" -- whitespace, ordering different
330+
331+jsonServer2 :: IO (Server JsonApi)
332+jsonServer2 = return $ return "{\"a\": 1,\"b\":[\"b\"]}"
333+
334+jsonServer3 :: IO (Server JsonApi)
335+jsonServer3 = return $ return "{\"a\": 2, \"b\": [\"b\"]}"
336+
337+jsonServer4 :: IO (Server JsonApi)
338+jsonServer4 = return $ return "{\"c\": 1, \"d\": [\"b\"]}"
339+
340+
341+ctx :: Context '[BasicAuthCheck ()]
342+ctx = BasicAuthCheck (const . return $ NoSuchUser) :. EmptyContext
68------------------------------------------------------------------------------
343------------------------------------------------------------------------------
69-- Utils
344-- Utils
70------------------------------------------------------------------------------
345------------------------------------------------------------------------------
346+evalExample :: (Example e, Arg e ~ ()) => e -> IO EvalResult
347+evalExample e = do
348+ r <- safeEvaluateExample e defaultParams ($ ()) progCallback
349+ case resultStatus r of
350+ Success -> return $ AllGood
351+ Failure _ reason -> return $ FailedWith $ show reason
352+ Pending {} -> error "should not happen"
353+ where
354+ progCallback _ = return ()
355+
356+data EvalResult
357+ = AnException SomeException
358+ | AllGood
359+ | FailedWith String
360+ deriving (Show)
361+
71
362
72args :: Args
363args :: Args
73−args = stdArgs { maxSuccess = noOfTestCases }
364+args = defaultArgs { maxSuccess = noOfTestCases }
74
365
75noOfTestCases :: Int
366noOfTestCases :: Int
76#if LONG_TESTS
367#if LONG_TESTS
77noOfTestCases = 20000
368noOfTestCases = 20000
78#else
369#else
79−noOfTestCases = 500
370+noOfTestCases = 1000
80#endif
371#endif