on master help sign in
direct
commit94b0f2e79ca4bd267556ae20137e4d824c1de96d authorMark Wotton <mwotton@gmail.com> dateSat 14 Nov 2020, 16:07 (6 years ago) parent8e47f92
6 files changed+278−216
package.yaml+3 roboservant.cabal+7−2 src/Roboservant.hs+2−2 src/Roboservant/Direct.hs+229 src/Roboservant/StateMachine.hs−176 test/Spec.hs+37−36
package.yaml+3
@@ -19,6 +19,9 @@ dependencies:
1919- bytestring
2020- containers
2121- hedgehog
22+- random
23+- lifted-base
24+- monad-control
2225- mtl
2326- servant # >= 0.17
2427- servant-client # >= 0.17
roboservant.cabal+7−2
@@ -4,7 +4,7 @@ cabal-version: 1.12
44--
55-- see: https://github.com/sol/hpack
66--
7−-- hash: eec4253f2196a058b05c08988d5c544f7238982abba63f6b0112014f043771b2
7+-- hash: 98b1024875f8d30a7818bbd16bf5f56375b76735f24931a17da3914852edcf6c
88
99name: roboservant
1010version: 0.1.0.2
@@ -32,7 +32,6 @@ library
3232 Roboservant
3333 Roboservant.Direct
3434 Roboservant.Hedgehog
35− Roboservant.StateMachine
3635 Roboservant.Types
3736 Roboservant.Types.Breakdown
3837 Roboservant.Types.FlattenServer
@@ -47,7 +46,10 @@ library
4746 , bytestring
4847 , containers
4948 , hedgehog
49+ , lifted-base
50+ , monad-control
5051 , mtl
52+ , random
5153 , servant
5254 , servant-client
5355 , servant-flatten
@@ -76,7 +78,10 @@ test-suite roboservant-test
7678 , hspec
7779 , hspec-core
7880 , hspec-hedgehog
81+ , lifted-base
82+ , monad-control
7983 , mtl
84+ , random
8085 , roboservant
8186 , servant
8287 , servant-client
src/Roboservant.hs+2−2
@@ -1,3 +1,3 @@
1−module Roboservant (module Roboservant.StateMachine) where
1+module Roboservant (module Roboservant.Direct) where
22
3−import Roboservant.StateMachine
3+import Roboservant.Direct
src/Roboservant/Direct.hsnew file+229
@@ -0,0 +1,229 @@
1+{-# LANGUAGE AllowAmbiguousTypes #-}
2+{-# LANGUAGE DataKinds #-}
3+{-# LANGUAGE DerivingStrategies #-}
4+{-# LANGUAGE FlexibleContexts #-}
5+{-# LANGUAGE FlexibleInstances #-}
6+{-# LANGUAGE GADTs #-}
7+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
8+{-# LANGUAGE LambdaCase #-}
9+{-# LANGUAGE PolyKinds #-}
10+{-# LANGUAGE RecordWildCards #-}
11+{-# LANGUAGE ScopedTypeVariables #-}
12+{-# LANGUAGE StandaloneDeriving #-}
13+{-# LANGUAGE TupleSections #-}
14+{-# LANGUAGE TypeApplications #-}
15+{-# LANGUAGE TypeFamilies #-}
16+{-# LANGUAGE TypeOperators #-}
17+{-# LANGUAGE UndecidableInstances #-}
18+
19+module Roboservant.Direct
20+ ( fuzz
21+ )
22+where
23+
24+import Control.Monad.Trans.Control
25+import Control.Monad.State.Strict
26+import Control.Exception.Lifted(throw,Exception,SomeException,catch)
27+import System.Random
28+import System.Timeout.Lifted
29+import Control.Monad (guard)
30+import Control.Monad.IO.Class (MonadIO, liftIO)
31+import Data.Dynamic (Dynamic, dynApply, dynTypeRep, fromDynamic)
32+import qualified Data.List.NonEmpty as NEL
33+import qualified Data.Map.Strict as Map
34+import Data.Map.Strict(Map)
35+import Data.Maybe (mapMaybe)
36+import Data.Typeable (TypeRep)
37+import Hedgehog
38+ ( Callback (Update),
39+ Command (Command),
40+ Concrete,
41+ MonadGen,
42+ Opaque (Opaque),
43+ PropertyT,
44+ Symbolic,
45+ Var,
46+ executeSequential,
47+ forAll,
48+ opaque,
49+ )
50+import qualified Hedgehog.Gen as Gen
51+import qualified Hedgehog.Range as Range
52+import Roboservant.Hedgehog (elementOrFail)
53+import Roboservant.Types
54+ ( ApiOffset (..),
55+ FlattenServer (..),
56+ Op (..),
57+ ReifiedApi,
58+ State (..),
59+ ToReifiedApi (..),
60+ emptyState,
61+ )
62+import Servant (Endpoints, Proxy (Proxy), Server, ServerError)
63+import Type.Reflection (SomeTypeRep)
64+
65+-- prop_concurrent,
66+import Control.Monad (guard)
67+import Control.Monad.IO.Class (MonadIO, liftIO)
68+import Data.Dynamic (Dynamic, dynApply, dynTypeRep, fromDynamic)
69+import qualified Data.List.NonEmpty as NEL
70+import qualified Data.Map.Strict as Map
71+import Data.Maybe (mapMaybe)
72+import Data.Typeable (TypeRep)
73+import qualified Hedgehog.Gen as Gen
74+import qualified Hedgehog.Range as Range
75+import Roboservant.Hedgehog (elementOrFail)
76+import Roboservant.Types
77+ ( ApiOffset (..),
78+ FlattenServer (..),
79+ Op (..),
80+ ReifiedApi,
81+ State (..),
82+ ToReifiedApi (..),
83+ emptyState,
84+ )
85+import Servant (Endpoints, Proxy (Proxy), Server, ServerError)
86+import Type.Reflection (SomeTypeRep)
87+
88+data RoboservantException
89+ = RoboservantException FailureType SomeException FuzzOp FuzzState
90+ deriving (Show)
91+-- we believe in nussink, lebowski
92+instance Exception RoboservantException
93+
94+data FailureType
95+ = ServerCrashed
96+ | CheckerFailed
97+ | NoPossibleMoves
98+ deriving (Show,Eq)
99+
100+-- update :: Callback Op (Opaque (NEL.NonEmpty Dynamic)) State
101+-- update = Update $ \s@State {..} op o' ->
102+-- case op of
103+-- Chewable offset tr v -> case _extract offset trs chewable of
104+-- Nothing -> error "internal error"
105+-- Just (tr, _extracted, rest) ->
106+-- s
107+-- { stateRefs =
108+-- -- so, what's going on here... we need to be able to grab and merge
109+-- -- a list of (typeref, var). this means that we need to be able to pull
110+-- -- out typereps symbolically too, we can't just rely on the dynamic being there.
111+-- -- Map.insertWith (<>) (dynTypeRep v) _ (pure o') stateRefs,
112+-- Map.insertWith (<>) tr (_ o') stateRefs,
113+-- chewable = rest
114+-- }
115+-- (Op (ApiOffset _offset) _args) ->
116+-- s
117+-- { chewable = _ o' : chewable
118+-- -- Map.insertWith
119+-- -- (<>)
120+-- -- tr
121+-- -- (pure o')
122+-- -- stateRefs
123+-- }
124+
125+
126+data Reference = Reference
127+ { tr :: TypeRep
128+ , offset :: Int
129+ } deriving (Show,Eq)
130+
131+data FuzzOp = FuzzOp ApiOffset [Reference]
132+ deriving (Show,Eq)
133+
134+data Config
135+ = Config
136+ { seed :: [Dynamic]
137+ , maxRuntime :: Int -- seconds to test for
138+ , rngSeed :: Int
139+ , currentRng :: StdGen
140+ }
141+
142+data FuzzState = FuzzState
143+ { path :: [FuzzOp]
144+ , stash :: Map TypeRep (NEL.NonEmpty (Reference,Dynamic))
145+ }
146+ deriving (Show)
147+
148+fuzz :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api))
149+ => Server api
150+ -> Config
151+ -> IO ()
152+ -> IO ()
153+fuzz server Config{..} checker =
154+ -- either we time out without finding an error, which is fine, or we find an error
155+ -- and throw an exception that propagates through this.
156+ void $ timeout (maxRuntime * 1000000) ( execStateT (forever go) FuzzState{..})
157+
158+ where
159+
160+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
161+
162+ elementOrFail :: MonadState FuzzState m
163+ => [a] -> m a
164+ elementOrFail = undefined
165+
166+ genOp :: MonadState FuzzState m
167+ => m FuzzOp
168+ genOp = do -- fs@FuzzState{..} = do
169+ -- choose a call to make, from the endpoints with fillable arguments.
170+ (offset, args) <- elementOrFail . options =<< get
171+ -- choose a (symbolic) argument from each. we drop the Dynamic here because it's
172+ -- not showable, which is no good for tests - though possibly it would make sense
173+ -- to use the constrained-dynamic stuff here? (or even just stash the string form)
174+ r <- (FuzzOp offset) <$> mapM (elementOrFail . fmap fst . NEL.toList) args
175+ modify' (\f -> f { path = r:path f })
176+ pure r
177+
178+
179+-- chooseOne opts = do
180+--
181+-- (offset,) <$> mapM elementOrFail args
182+ where
183+ options :: FuzzState -> [(ApiOffset, [NEL.NonEmpty (Reference, Dynamic)])]
184+ options FuzzState{..} =
185+ mapMaybe
186+ ( \(offset, (argreps, _dynCall)) -> (offset,) <$> do
187+ mapM (flip Map.lookup stash) argreps
188+ )
189+ reifiedApi
190+ execute :: MonadState FuzzState m
191+ => FuzzOp -> m ()
192+ execute = undefined
193+
194+
195+
196+ -- execute (Op (ApiOffset offset) args) = do
197+ -- fmap Opaque . liftIO $ do
198+ -- let realArgs = map opaque args
199+ -- let (_offset, (_staticArgs, endpoint)) = staticRoutes !! offset
200+ -- -- now, magic happens: we apply some dynamic arguments to a dynamic
201+ -- -- function and hopefully something useful pops out the end.
202+ -- func = foldr (\arg curr -> flip dynApply arg =<< curr) (Just endpoint) realArgs
203+ -- let showable = "blah" -- map dynTypeRep (_endpoint : realArgs)
204+ -- case func of
205+ -- Nothing -> error ("all screwed up: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
206+ -- Just (f') -> do
207+ -- case fromDynamic f' of
208+ -- Nothing -> error ("all screwed up: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
209+ -- Just f -> liftIO f >>= \case
210+ -- Left (serverError :: ServerError) -> error (show serverError)
211+ -- Right (_typeRep :: SomeTypeRep, (dyn :: NEL.NonEmpty Dynamic)) -> pure dyn
212+
213+
214+ go :: (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m)
215+ => m ()
216+ go = do
217+ fuzzOp <- genOp
218+ catch (execute fuzzOp)
219+ (\(e :: SomeException) -> throw . RoboservantException ServerCrashed e fuzzOp =<< get)
220+ catch (liftIO checker)
221+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed e fuzzOp =<< get)
222+
223+ -- actions <-
224+ -- forAll $ do
225+ -- Gen.sequential
226+ -- (Range.linear 1 100)
227+ -- emptyState
228+ -- (fmap preload seed <> [callEndpoint reifiedApi])
229+ -- executeSequential emptyState actions
src/Roboservant/StateMachine.hsdeleted−176
@@ -1,176 +0,0 @@
1−{-# LANGUAGE AllowAmbiguousTypes #-}
2−{-# LANGUAGE DataKinds #-}
3−{-# LANGUAGE DerivingStrategies #-}
4−{-# LANGUAGE FlexibleContexts #-}
5−{-# LANGUAGE FlexibleInstances #-}
6−{-# LANGUAGE GADTs #-}
7−{-# LANGUAGE GeneralizedNewtypeDeriving #-}
8−{-# LANGUAGE LambdaCase #-}
9−{-# LANGUAGE PolyKinds #-}
10−{-# LANGUAGE RecordWildCards #-}
11−{-# LANGUAGE ScopedTypeVariables #-}
12−{-# LANGUAGE StandaloneDeriving #-}
13−{-# LANGUAGE TupleSections #-}
14−{-# LANGUAGE TypeApplications #-}
15−{-# LANGUAGE TypeFamilies #-}
16−{-# LANGUAGE TypeOperators #-}
17−{-# LANGUAGE UndecidableInstances #-}
18−
19−module Roboservant.StateMachine
20− ( prop_sequential,
21− )
22−where
23−
24−-- prop_concurrent,
25−
26−import Control.Monad (guard)
27−import Control.Monad.IO.Class (MonadIO, liftIO)
28−import Data.Dynamic (Dynamic, dynApply, dynTypeRep, fromDynamic)
29−import qualified Data.List.NonEmpty as NEL
30−import qualified Data.Map.Strict as Map
31−import Data.Maybe (mapMaybe)
32−import Data.Typeable (TypeRep)
33−import Hedgehog
34− ( Callback (Update),
35− Command (Command),
36− Concrete,
37− MonadGen,
38− Opaque (Opaque),
39− PropertyT,
40− Symbolic,
41− Var,
42− executeSequential,
43− forAll,
44− opaque,
45− )
46−import qualified Hedgehog.Gen as Gen
47−import qualified Hedgehog.Range as Range
48−import Roboservant.Hedgehog (elementOrFail)
49−import Roboservant.Types
50− ( ApiOffset (..),
51− FlattenServer (..),
52− Op (..),
53− ReifiedApi,
54− State (..),
55− ToReifiedApi (..),
56− emptyState,
57− )
58−import Servant (Endpoints, Proxy (Proxy), Server, ServerError)
59−import Type.Reflection (SomeTypeRep)
60−
61−callEndpoint :: (MonadGen n, MonadIO m) => ReifiedApi -> Command n m State
62−callEndpoint staticRoutes =
63− let gen :: MonadGen n => State Symbolic -> Maybe (n (Op Symbolic))
64− gen State {..} = case chewable of
65− (_ : _) -> Just $ do
66− (offset, v) <- Gen.element (zip [0 ..] chewable)
67− pure $ Chewable offset _ _
68− _ -> do
69− guard $ not $ null options
70− pure $ uncurry Op <$> chooseOne options
71− where
72− chooseOne ::
73− MonadGen n =>
74− [ ( a,
75− [[b]]
76− )
77− ] ->
78− n
79− ( a,
80− [b]
81− )
82− chooseOne opts = do
83− (offset, args) <- elementOrFail opts
84− (offset,) <$> mapM elementOrFail args
85− options :: [(ApiOffset, [[Var (Opaque Dynamic) Symbolic]])]
86− options =
87− mapMaybe
88− ( \(offset, (argreps, _dynCall)) -> (offset,) <$> do
89− mapM fillableWith argreps
90− )
91− staticRoutes
92− fillableWith :: TypeRep -> Maybe [Var (Opaque Dynamic) Symbolic]
93− fillableWith tr = NEL.toList <$> Map.lookup tr stateRefs
94− execute ::
95− (MonadIO m) =>
96− Op Concrete ->
97− m (Opaque (NEL.NonEmpty Dynamic))
98− -- bit subtle here - a chewable v _also_ gets a var, because chewing it should
99− -- only give subcomponents.
100− execute (Chewable offset trs v) = _ -- pure . Opaque $ opaque $ pure v -- pure $ Opaque v -- <$> liftIO (newIORef v)
101− execute (Op (ApiOffset offset) args) = do
102− fmap Opaque . liftIO $ do
103− let realArgs = map opaque args
104− let (_offset, (_staticArgs, endpoint)) = staticRoutes !! offset
105− -- now, magic happens: we apply some dynamic arguments to a dynamic
106− -- function and hopefully something useful pops out the end.
107− func = foldr (\arg curr -> flip dynApply arg =<< curr) (Just endpoint) realArgs
108− let showable = "blah" -- map dynTypeRep (_endpoint : realArgs)
109− case func of
110− Nothing -> error ("all screwed up: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
111− Just (f') -> do
112− case fromDynamic f' of
113− Nothing -> error ("all screwed up: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
114− Just f -> liftIO f >>= \case
115− Left (serverError :: ServerError) -> error (show serverError)
116− Right (_typeRep :: SomeTypeRep, (dyn :: NEL.NonEmpty Dynamic)) -> pure dyn
117− in Command
118− gen
119− execute
120− [ update
121− ]
122−
123−update :: Callback Op (Opaque (NEL.NonEmpty Dynamic)) State
124−update = Update $ \s@State {..} op o' ->
125− case op of
126− Chewable offset tr v -> case _extract offset trs chewable of
127− Nothing -> error "internal error"
128− Just (tr, _extracted, rest) ->
129− s
130− { stateRefs =
131− -- so, what's going on here... we need to be able to grab and merge
132− -- a list of (typeref, var). this means that we need to be able to pull
133− -- out typereps symbolically too, we can't just rely on the dynamic being there.
134− -- Map.insertWith (<>) (dynTypeRep v) _ (pure o') stateRefs,
135− Map.insertWith (<>) tr (_ o') stateRefs,
136− chewable = rest
137− }
138− (Op (ApiOffset _offset) _args) ->
139− s
140− { chewable = _ o' : chewable
141− -- Map.insertWith
142− -- (<>)
143− -- tr
144− -- (pure o')
145− -- stateRefs
146− }
147−
148−extract :: Int -> [a] -> Maybe (a, [a])
149−extract = undefined
150−
151−preload :: (MonadGen n, MonadIO m) => Dynamic -> Command n m State
152−preload = undefined
153−
154−prop_sequential :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api)) => Server api -> [Dynamic] -> PropertyT IO ()
155−prop_sequential server seed = do
156− let reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
157− actions <-
158− forAll $ do
159− Gen.sequential
160− (Range.linear 1 100)
161− emptyState
162− (fmap preload seed <> [callEndpoint reifiedApi])
163− executeSequential emptyState actions
164−-- prop_concurrent :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api)) => Server api -> [Dynamic] -> PropertyT IO ()
165−-- prop_concurrent server seed =
166−-- let reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
167−-- in do
168−-- actions <-
169−-- forAll $
170−-- Gen.parallel
171−-- (Range.linear 1 50)
172−-- (Range.linear 1 10)
173−-- (emptyState {chewable = seed})
174−-- [callEndpoint reifiedApi]
175−-- test $
176−-- executeParallel (emptyState {chewable = seed}) actions
test/Spec.hs+37−36
@@ -2,15 +2,15 @@
22{-# LANGUAGE OverloadedStrings #-}
33{-# LANGUAGE TypeApplications #-}
44
5−import qualified Foo
6−import qualified Seeded
7−import qualified Headers
5+-- import qualified Foo
6+-- import qualified Seeded
7+-- import qualified Headers
88import qualified UnsafeIO
99import qualified Roboservant as RS
1010
11−import Test.Hspec.Hedgehog
11+-- import Test.Hspec.Hedgehog
1212import Test.Hspec
13−import Test.Hspec.Core.Spec(shouldFail)
13+-- import Test.Hspec.Core.Spec(shouldFail)
1414
1515import Data.Dynamic(toDyn)
1616
@@ -20,34 +20,35 @@ main = hspec spec
2020spec :: Spec
2121spec = do
2222 describe "Basic usage" $ do
23− describe "seeded" $ do
24− modifyMaxSuccess (const 10000) $
25− shouldFail $
26− it "finds an error using information passed in" $
27− hedgehog $ RS.prop_sequential @Seeded.Api Seeded.server [toDyn $ Seeded.Seed 1]
28−
29− modifyMaxSuccess (const 10000) $
30−
31− shouldFail $ it "finds an error in a basic app" $
32− hedgehog $ RS.prop_sequential @Foo.Api Foo.server []
33−
34− describe "Headers" $ do
35− modifyMaxSuccess (const 10000) $
36−
37− shouldFail $ it "should find a failure that's dependent on using header info" $ do
38− hedgehog $ RS.prop_sequential @Headers.Api Headers.server []
39− -- pending
40−
41− -- -- The UnsafeIO checker does not actually really use the contextually aware stuff, though it
42− -- -- could: it's mostly here to show how to test for concurrency problems.
43− describe "concurrency bugs" $ do
44− before UnsafeIO.makeServer $ do
45− describe "sequential checking" $ do
46− it "safe use" $ \unsafeServer -> do
47− hedgehog $ RS.prop_sequential @UnsafeIO.UnsafeApi unsafeServer []
48−
49− modifyMaxSuccess (const 10000) $
50− shouldFail $
51− describe "concurrent" $ do
52− it "concurrent, dangerous use" $ \unsafeServer -> do
53− RS.prop_concurrent @UnsafeIO.UnsafeApi unsafeServer []
23+ pure ()
24+ -- describe "seeded" $ do
25+ -- modifyMaxSuccess (const 10000) $
26+ -- shouldFail $
27+ -- it "finds an error using information passed in" $
28+ -- hedgehog $ RS.prop_sequential @Seeded.Api Seeded.server [toDyn $ Seeded.Seed 1]
29+
30+ -- modifyMaxSuccess (const 10000) $
31+
32+ -- shouldFail $ it "finds an error in a basic app" $
33+ -- hedgehog $ RS.prop_sequential @Foo.Api Foo.server []
34+
35+ -- describe "Headers" $ do
36+ -- modifyMaxSuccess (const 10000) $
37+
38+ -- shouldFail $ it "should find a failure that's dependent on using header info" $ do
39+ -- hedgehog $ RS.prop_sequential @Headers.Api Headers.server []
40+ -- -- pending
41+
42+ -- -- -- The UnsafeIO checker does not actually really use the contextually aware stuff, though it
43+ -- -- -- could: it's mostly here to show how to test for concurrency problems.
44+ -- describe "concurrency bugs" $ do
45+ -- before UnsafeIO.makeServer $ do
46+ -- describe "sequential checking" $ do
47+ -- it "safe use" $ \unsafeServer -> do
48+ -- hedgehog $ RS.prop_sequential @UnsafeIO.UnsafeApi unsafeServer []
49+
50+ -- modifyMaxSuccess (const 10000) $
51+ -- shouldFail $
52+ -- describe "concurrent" $ do
53+ -- it "concurrent, dangerous use" $ \unsafeServer -> do
54+ -- RS.prop_concurrent @UnsafeIO.UnsafeApi unsafeServer []