@@ -2,53 +2,104 @@
2{-# LANGUAGE FlexibleContexts #-}
2{-# LANGUAGE FlexibleContexts #-}
3{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
4{-# LANGUAGE LambdaCase #-}
4{-# LANGUAGE LambdaCase #-}
5−{-# LANGUAGE PolyKinds #-}
6{-# LANGUAGE RankNTypes #-}
5{-# LANGUAGE RankNTypes #-}
7{-# LANGUAGE RecordWildCards #-}
6{-# LANGUAGE RecordWildCards #-}
8{-# LANGUAGE ScopedTypeVariables #-}
7{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE TupleSections #-}
8{-# LANGUAGE TupleSections #-}
10−
9+{-# LANGUAGE TypeApplications #-}
11{-# LANGUAGE TypeFamilies #-}
10{-# LANGUAGE TypeFamilies #-}
12{-# LANGUAGE UndecidableInstances #-}
12{-# LANGUAGE UndecidableInstances #-}
14module Roboservant.Direct
14module Roboservant.Direct
16− Config (..),
15+ ( fuzz, Config(..)
17− -- TODO come up with something smarter than exporting all this, we should
17+ -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
18+ -- have some nice error-display functions
19− RoboservantException (..),
19+ , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
27import Control.Exception.Lifted
23import Control.Exception.Lifted
28− ( Exception,
27+ Exception,
32+import Control.Exception.Lifted
29− Handler (Handler),
34+ Handler(..),
40+import Control.Exception.Lifted
30− SomeAsyncException,
43+ SomeAsyncException,
31− SomeException,
44+ SomeException,
37import Control.Monad.State.Strict
50import Control.Monad.State.Strict
38− ( MonadIO (..),
51+ ( MonadState, MonadIO, get, modify', liftIO, runStateT )
39− MonadState (get),
52+import Control.Monad.State.Strict
40− StateT (runStateT),
53+ ( MonadState, MonadIO, get, modify', liftIO, execStateT )
41− modify',
54+import Control.Monad.State.Strict
42− )
55+ ( MonadIO, MonadState, get, liftIO, modify', runStateT )
57+import Control.Exception.Lifted(throw,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
58+import Control.Monad(void,replicateM,forM_)
59+import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,execStateT)
60+import Control.Monad.Trans.Control(MonadBaseControl)
61+import Data.Dependent.Map (DMap)
62+import Data.Dynamic (Dynamic(..), dynApply, dynTypeRep, fromDynamic)
63+import Data.List.NonEmpty (NonEmpty(..))
64+import Data.Map.Strict(Map)
65+import qualified Data.Set as Set
66+import Data.Maybe (mapMaybe)
67+import Data.Typeable (TypeRep)
68+import Servant ( Endpoints, Proxy(Proxy), Server, ServerError(..) )
69+import Servant ( Endpoints, Proxy(Proxy), Server, ServerError(..) )
71+import System.Random ( StdGen, randomR, mkStdGen )
72+import System.Random ( StdGen, mkStdGen, randomR )
74+import qualified Data.List.NonEmpty as NEL
75+import qualified Data.Map.Strict as Map
76+import Data.Time.Clock
77+import Data.Time.Clock
79+import Data.List(sortOn)
80+import Control.Arrow(second)
81+import System.Timeout.Lifted(timeout)
82+import GHC.Generics ( (:*:)(..) )
83+import GHC.Generics ( (:*:)(..) )
85+import qualified Data.Vinyl.Functor as V
86+import qualified Data.Vinyl.Functor as V
88+import qualified Data.Vinyl.Curry as V
89+import qualified Data.Vinyl.Curry as V
93+import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
94+import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
43import Control.Monad.Trans.Control (MonadBaseControl)
95import Control.Monad.Trans.Control (MonadBaseControl)
44import qualified Data.Dependent.Map as DM
96import qualified Data.Dependent.Map as DM
45−import Data.Dynamic (Dynamic (..))
97+import Data.Dynamic (Dynamic (..), dynTypeRep)
46−import qualified Data.IntSet as IntSet
47import Data.List.NonEmpty (NonEmpty (..))
98import Data.List.NonEmpty (NonEmpty (..))
48import qualified Data.List.NonEmpty as NEL
99import qualified Data.List.NonEmpty as NEL
49import Data.Maybe (mapMaybe)
100import Data.Maybe (mapMaybe)
50import qualified Data.Set as Set
101import qualified Data.Set as Set
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
102+import Data.Time.Clock
52import qualified Data.Vinyl as V
103import qualified Data.Vinyl as V
53import qualified Data.Vinyl.Curry as V
104import qualified Data.Vinyl.Curry as V
54import qualified Data.Vinyl.Functor as V
105import qualified Data.Vinyl.Functor as V
@@ -56,26 +107,25 @@ import GHC.Generics ((:*:) (..))
56import Roboservant.Types
107import Roboservant.Types
57 ( ApiOffset (..),
108 ( ApiOffset (..),
58 Argument (..),
109 Argument (..),
59− InteractionError(..),
110+ FlattenServer (..),
60 Provenance (..),
111 Provenance (..),
62 ReifiedEndpoint (..),
112 ReifiedEndpoint (..),
63 Stash (..),
113 Stash (..),
64 StashValue (..),
114 StashValue (..),
67−import Roboservant.Types.Config
118+import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
69−import System.Random (Random (randomR), StdGen, mkStdGen)
119+import System.Random (StdGen, mkStdGen, randomR)
70import qualified Type.Reflection as R
120import qualified Type.Reflection as R
121+import qualified Data.IntSet as IntSet
72data RoboservantException
123data RoboservantException
73 = RoboservantException
124 = RoboservantException
74− { failureReason :: FailureType,
125+ { failureReason :: FailureType
75− serverException :: Maybe SomeException,
126+ , serverException :: Maybe SomeException
76− fuzzState :: FuzzState
127+ , fuzzState :: FuzzState
78− deriving (Show)
128+ } deriving (Show)
80instance Exception RoboservantException
130instance Exception RoboservantException
@@ -84,7 +134,7 @@ data FailureType
84 | CheckerFailed
134 | CheckerFailed
85 | NoPossibleMoves
135 | NoPossibleMoves
86 | InsufficientCoverage Double
136 | InsufficientCoverage Double
87− deriving (Show, Eq)
137+ deriving (Show,Eq)
89data FuzzOp
139data FuzzOp
@@ -93,6 +143,15 @@ data FuzzOp
94 deriving (Show, Eq)
144 deriving (Show, Eq)
148+ { seed :: [(Dynamic,Int)],
149+ maxRuntime :: Double, -- seconds to test for
152+ coverageThreshold :: Double
96data FuzzState
155data FuzzState
97 = FuzzState
156 = FuzzState
98 { path :: [FuzzOp],
157 { path :: [FuzzOp],
@@ -105,7 +164,7 @@ data EndpointOption
105 = forall as.
164 = forall as.
106 (V.RecordToList as, V.RMap as) =>
165 (V.RecordToList as, V.RMap as) =>
107 EndpointOption
166 EndpointOption
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
167+ { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))),
109 eoArgs :: V.Rec (TypedF StashValue) as
168 eoArgs :: V.Rec (TypedF StashValue) as
@@ -114,25 +173,19 @@ data StopReason
114 | HitMaxIterations
173 | HitMaxIterations
115 deriving (Show, Eq)
174 deriving (Show, Eq)
117−data Report
176+data Report = Report
119− { textual :: String,
177+ { textual :: String
120− rsException :: RoboservantException
178+ , rsException :: RoboservantException}
122 deriving (Show)
179 deriving (Show)
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127−-- fuzzClient = undefined
132− ReifiedApi ->
182+ forall api.
183+ (FlattenServer api, ToReifiedApi (Endpoints api)) =>
133 Config ->
185 Config ->
134 IO (Maybe Report)
187 IO (Maybe Report)
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
188+fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
136 let path = []
189 let path = []
137 stash = addToStash seed mempty
190 stash = addToStash seed mempty
138 currentRng = mkStdGen rngSeed
191 currentRng = mkStdGen rngSeed
@@ -141,36 +194,39 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
141 runStateT
194 runStateT
142 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
195 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
143 FuzzState {..}
196 FuzzState {..}
144− logInfo $ show stopreason
197+ print stopreason
145 pure Nothing
198 pure Nothing
147 -- something less terrible later
200 -- something less terrible later
148 formatException :: RoboservantException -> Report
201 formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
202+ formatException r@(RoboservantException failureType exception _state) = Report
151− (unlines [show failureType, show exception])
203+ (unlines [ show failureType , show exception ])
153− displayDiagnostics FuzzState {..} = liftIO $
206+ displayDiagnostics FuzzState{..} = liftIO $ do
155− ["api endpoints covered"]
207+ putStrLn "api endpoints covered"
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
208+ mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
210+ putStrLn "types in stash"
211+ DM.forWithKey_ (getStash stash) $ \k v ->
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
212+ print (NEL.length $ getStashValue v)
214+ -- (print . second NEL.length)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
215+ -- (sortOn (show . fst) $ Map.toList $ getStash stash)
161− -- (show . NEL.length $ getStashValue v))
217+ -- evaluateCoverage :: FuzzState -> m ()
163− evaluateCoverage f@FuzzState {..}
218+ evaluateCoverage f@FuzzState{..}
164 | coverage > coverageThreshold = pure ()
219 | coverage > coverageThreshold = pure ()
165 | otherwise = do
220 | otherwise = do
166− displayDiagnostics f
221+ displayDiagnostics f
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
222+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
223+ where hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
170− totalRoutes = fromIntegral routeCount
224+ totalRoutes = fromIntegral routeCount
171− coverage = hitRoutes / totalRoutes
225+ coverage = hitRoutes / totalRoutes
172− untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
228+ untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
173− untilDone (0, _) _ = pure HitMaxIterations
229+ untilDone (0,_) _ = pure HitMaxIterations
174 untilDone (n, deadline) action = do
230 untilDone (n, deadline) action = do
175 now <- liftIO getCurrentTime
231 now <- liftIO getCurrentTime
176 if now > deadline
232 if now > deadline
@@ -178,13 +234,12 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
179 _ <- action
235 _ <- action
180 untilDone (n -1, deadline) action
236 untilDone (n -1, deadline) action
181−
237+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
182 routeCount = length reifiedApi
238 routeCount = length reifiedApi
183− elementOrFail ::
239+
184− (MonadState FuzzState m, MonadIO m) =>
240+ elementOrFail :: (MonadState FuzzState m, MonadIO m)
186− m a
241+ => [a] -> m a
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
242+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
188 elementOrFail l = do
243 elementOrFail l = do
189 st <- get
244 st <- get
190 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
245 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
@@ -195,7 +250,7 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
195 ( forall as.
250 ( forall as.
196 (V.RecordToList as, V.RMap as) =>
251 (V.RecordToList as, V.RMap as) =>
197 FuzzOp ->
252 FuzzOp ->
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
253+ V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))) ->
199 V.Rec (TypedF V.Identity) as ->
254 V.Rec (TypedF V.Identity) as ->
@@ -236,53 +291,57 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
236 execute ::
291 execute ::
237 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
292 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
238 FuzzOp ->
293 FuzzOp ->
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
294+ V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))) ->
240 V.Rec (TypedF V.Identity) as ->
295 V.Rec (TypedF V.Identity) as ->
242 execute fuzzop func args = do
297 execute fuzzop func args = do
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
298+ (liftIO . print . (fuzzop,) . stash ) =<< get
300+ -- let showable = unlines $ ("args":map show argTypes)
304+ -- -- ,show (dynTypeRep dyncall)
307+ -- liftIO $ putStrLn showable
244 liftIO (V.runcurry' func argVals) >>= \case
308 liftIO (V.runcurry' func argVals) >>= \case
245− Left (e::InteractionError) ->
309+ -- parameterise this
246− if fatalError e
310+ Left (serverError :: ServerError) ->
247− then throw e
311+ case errHTTPCode serverError of
248− else pure ()
312+ 500 -> throw serverError
314+ liftIO $ print ("ignoring non-500 error", serverError)
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
315+ Right (dyn :: NEL.NonEmpty (Dynamic,Int)) -> do
250− modify'
316+ -- liftIO $ print ("storing", fmap dynTypeRep dyn)
251− ( \fs@FuzzState {..} ->
317+ modify' (\fs@FuzzState{..} ->
252− fs {stash = addToStash (NEL.toList dyn) stash}
318+ fs { stash = addToStash (NEL.toList dyn) stash } )
255 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
320 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
321+ argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
258 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
323 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
260 go = withOp $ \op func args -> do
325 go = withOp $ \op func args -> do
262− (execute op func args)
326+ catches (execute op func args)
263− [ Handler (\(e :: SomeAsyncException) -> throw e),
327+ [ Handler (\(e :: SomeAsyncException) -> throw e)
265− ( \(e :: SomeException) ->
328+ , Handler (\(e :: SomeException) -> do
329+ displayDiagnostics =<< get
266− throw . RoboservantException ServerCrashed (Just e) =<< get
330+ throw . RoboservantException ServerCrashed (Just e) =<< get)
269− catch
332+ catch (liftIO checker)
270− (liftIO healthCheck)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
333+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
273addToStash ::
335addToStash ::
274− [(Dynamic, Int)] ->
336+ [(Dynamic,Int)] ->
277addToStash result stash =
339addToStash result stash =
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
341+ ( \(Dynamic tr x,hashed) (Stash dict) ->
281− DM.insertWith
343+ DM.insertWith renumber tr
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
344+ (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed)) dict
@@ -295,11 +354,11 @@ addToStash result stash =
295 | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
354 | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
296 | otherwise =
355 | otherwise =
297 StashValue
356 StashValue
298− ( case NEL.toList singleDyn of
357+ (case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
358+ [([Provenance tr _], dyn)] ->
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
359+ l <> pure (([Provenance tr (length (NEL.last l) + 1)], dyn))
301− _ -> error "should be impossible"
360+ _ -> error "should be impossible")
303 (IntSet.union singleHash intSet)
362 (IntSet.union singleHash intSet)
305-- why isn't this in vinyl?
364-- why isn't this in vinyl?