@@ -2,33 +2,53 @@
2{-# LANGUAGE FlexibleContexts #-}
2{-# LANGUAGE FlexibleContexts #-}
3{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
4{-# LANGUAGE LambdaCase #-}
4{-# LANGUAGE LambdaCase #-}
5+{-# LANGUAGE PolyKinds #-}
5{-# LANGUAGE RankNTypes #-}
6{-# LANGUAGE RankNTypes #-}
6{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE ScopedTypeVariables #-}
8{-# LANGUAGE ScopedTypeVariables #-}
8{-# LANGUAGE TupleSections #-}
9{-# LANGUAGE TupleSections #-}
9−{-# LANGUAGE TypeApplications #-}
10+
11+{-# LANGUAGE TypeFamilies #-}
12+{-# LANGUAGE UndecidableInstances #-}
11module Roboservant.Direct
14module Roboservant.Direct
13 Config (..),
16 Config (..),
14 -- TODO come up with something smarter than exporting all this, we should
17 -- TODO come up with something smarter than exporting all this, we should
15 -- have some nice error-display functions
18 -- have some nice error-display functions
16 RoboservantException (..),
19 RoboservantException (..),
17 FuzzState (..),
20 FuzzState (..),
18 FuzzOp (..),
21 FuzzOp (..),
22−import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
27+import Control.Exception.Lifted
23−import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
37+import Control.Monad.State.Strict
24import Control.Monad.Trans.Control (MonadBaseControl)
43import Control.Monad.Trans.Control (MonadBaseControl)
25import qualified Data.Dependent.Map as DM
44import qualified Data.Dependent.Map as DM
26−import Data.Dynamic (Dynamic (..), dynTypeRep)
45+import Data.Dynamic (Dynamic (..))
46+import qualified Data.IntSet as IntSet
27import Data.List.NonEmpty (NonEmpty (..))
47import Data.List.NonEmpty (NonEmpty (..))
28import qualified Data.List.NonEmpty as NEL
48import qualified Data.List.NonEmpty as NEL
29import Data.Maybe (mapMaybe)
49import Data.Maybe (mapMaybe)
30import qualified Data.Set as Set
50import qualified Data.Set as Set
31−import Data.Time.Clock
51+import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
32import qualified Data.Vinyl as V
52import qualified Data.Vinyl as V
33import qualified Data.Vinyl.Curry as V
53import qualified Data.Vinyl.Curry as V
34import qualified Data.Vinyl.Functor as V
54import qualified Data.Vinyl.Functor as V
@@ -36,30 +56,34 @@ import GHC.Generics ((:*:) (..))
36import Roboservant.Types
56import Roboservant.Types
37 ( ApiOffset (..),
57 ( ApiOffset (..),
38 Argument (..),
58 Argument (..),
39− FlattenServer (..),
59+ InteractionError(..),
40 Provenance (..),
60 Provenance (..),
41 ReifiedEndpoint (..),
62 ReifiedEndpoint (..),
42 Stash (..),
63 Stash (..),
43 StashValue (..),
64 StashValue (..),
47−import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
67+import Roboservant.Types.Config
48−import System.Random (StdGen, mkStdGen, randomR)
69+import System.Random (Random (randomR), StdGen, mkStdGen)
49import qualified Type.Reflection as R
70import qualified Type.Reflection as R
51data RoboservantException
72data RoboservantException
52− = RoboservantException FailureType (Maybe SomeException) Int FuzzState
73+ = RoboservantException
74+ { failureReason :: FailureType,
75+ serverException :: Maybe SomeException,
76+ fuzzState :: FuzzState
53 deriving (Show)
78 deriving (Show)
55−-- we believe in nussink, lebowski
56instance Exception RoboservantException
80instance Exception RoboservantException
58data FailureType
82data FailureType
59 = ServerCrashed
83 = ServerCrashed
60 | CheckerFailed
84 | CheckerFailed
61 | NoPossibleMoves
85 | NoPossibleMoves
62− | InsufficientCoverage
86+ | InsufficientCoverage Double
63 deriving (Show, Eq)
87 deriving (Show, Eq)
65data FuzzOp
89data FuzzOp
@@ -69,15 +93,6 @@ data FuzzOp
70 deriving (Show, Eq)
94 deriving (Show, Eq)
75− maxRuntime :: Integer, -- seconds to test for
78− coverageThreshold :: Double
81data FuzzState
96data FuzzState
82 = FuzzState
97 = FuzzState
83 { path :: [FuzzOp],
98 { path :: [FuzzOp],
@@ -90,7 +105,7 @@ data EndpointOption
90 = forall as.
105 = forall as.
91 (V.RecordToList as, V.RMap as) =>
106 (V.RecordToList as, V.RMap as) =>
92 EndpointOption
107 EndpointOption
93− { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty Dynamic))),
108+ { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
94 eoArgs :: V.Rec (TypedF StashValue) as
109 eoArgs :: V.Rec (TypedF StashValue) as
@@ -101,40 +116,59 @@ data StopReason
102data Report
117data Report
104− {textual :: String}
119+ { textual :: String,
120+ rsException :: RoboservantException
105− deriving (Show, Eq)
122+ deriving (Show)
126+-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127+-- fuzzClient = undefined
108− forall api.
132+ ReifiedApi ->
109− (FlattenServer api, ToReifiedApi (Endpoints api)) =>
111 Config ->
133 Config ->
113 IO (Maybe Report)
134 IO (Maybe Report)
114−fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
135+fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
115 let path = []
136 let path = []
116 stash = addToStash seed mempty
137 stash = addToStash seed mempty
117 currentRng = mkStdGen rngSeed
138 currentRng = mkStdGen rngSeed
118− deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
139+ deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
119 (stopreason, _fs) <-
140 (stopreason, _fs) <-
120 runStateT
141 runStateT
121 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
142 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
122 FuzzState {..}
143 FuzzState {..}
123− print stopreason
144+ logInfo $ show stopreason
124 pure Nothing
145 pure Nothing
126 -- something less terrible later
147 -- something less terrible later
127 formatException :: RoboservantException -> Report
148 formatException :: RoboservantException -> Report
128− formatException = Report . show
149+ formatException r@(RoboservantException failureType exception _state) =
129− -- evaluateCoverage :: FuzzState -> m ()
150+ Report
151+ (unlines [show failureType, show exception])
153+ displayDiagnostics FuzzState {..} = liftIO $
155+ ["api endpoints covered"]
156+ <> (map show . Set.toList . Set.fromList $ map apiOffset path)
157+ <> ["", "types in stash"]
158+ <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
159+ -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
161+ -- (show . NEL.length $ getStashValue v))
130 evaluateCoverage f@FuzzState {..}
163 evaluateCoverage f@FuzzState {..}
131− | hitRoutes / totalRoutes > coverageThreshold = do
164+ | coverage > coverageThreshold = pure ()
132− liftIO $ print ("passed coverage", hitRoutes, totalRoutes)
165+ | otherwise = do
133− pure ()
166+ displayDiagnostics f
134− | otherwise = throw $ RoboservantException InsufficientCoverage Nothing routeCount f
167+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
136− hitRoutes = (fromIntegral . Set.size . Set.fromList $ map apiOffset path)
169+ hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
137− totalRoutes = (fromIntegral routeCount)
170+ totalRoutes = fromIntegral routeCount
171+ coverage = hitRoutes / totalRoutes
138 untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
172 untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
139 untilDone (0, _) _ = pure HitMaxIterations
173 untilDone (0, _) _ = pure HitMaxIterations
140 untilDone (n, deadline) action = do
174 untilDone (n, deadline) action = do
@@ -144,13 +178,13 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
145 _ <- action
179 _ <- action
146 untilDone (n -1, deadline) action
180 untilDone (n -1, deadline) action
147− reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
181+
148 routeCount = length reifiedApi
182 routeCount = length reifiedApi
149 elementOrFail ::
183 elementOrFail ::
150 (MonadState FuzzState m, MonadIO m) =>
184 (MonadState FuzzState m, MonadIO m) =>
153− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing routeCount =<< get
187+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
154 elementOrFail l = do
188 elementOrFail l = do
155 st <- get
189 st <- get
156 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
190 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
@@ -161,7 +195,7 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
161 ( forall as.
195 ( forall as.
162 (V.RecordToList as, V.RMap as) =>
196 (V.RecordToList as, V.RMap as) =>
163 FuzzOp ->
197 FuzzOp ->
164− V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
198+ V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
165 V.Rec (TypedF V.Identity) as ->
199 V.Rec (TypedF V.Identity) as ->
@@ -171,7 +205,7 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
171 (offset, EndpointOption {..}) <- elementOrFail . options =<< get
205 (offset, EndpointOption {..}) <- elementOrFail . options =<< get
173 V.rtraverse
207 V.rtraverse
174− ( \(tr :*: StashValue svs) ->
208+ ( \(tr :*: StashValue svs _) ->
175 elementOrFail $
209 elementOrFail $
177 (\i xy -> V.Const i :*: tr :*: xy)
211 (\i xy -> V.Const i :*: tr :*: xy)
@@ -202,39 +236,24 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
202 execute ::
236 execute ::
203 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
237 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
204 FuzzOp ->
238 FuzzOp ->
205− V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
239+ V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
206 V.Rec (TypedF V.Identity) as ->
240 V.Rec (TypedF V.Identity) as ->
208 execute fuzzop func args = do
242 execute fuzzop func args = do
209− liftIO $ print fuzzop
243+ (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
213− ("args" : map show argTypes)
217− -- ,show (dynTypeRep dyncall)
221− liftIO $ putStrLn showable
222 liftIO (V.runcurry' func argVals) >>= \case
244 liftIO (V.runcurry' func argVals) >>= \case
223− -- parameterise this
245+ Left (e::InteractionError) ->
224− Left (serverError :: ServerError) ->
246+ if fatalError e
225− case errHTTPCode serverError of
247+ then throw e
226− 500 -> throw serverError
248+ else pure ()
228− liftIO $ print ("ignoring non-500 error", serverError)
229− Right (dyn :: NEL.NonEmpty Dynamic) -> do
249+ Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
230− liftIO $ print ("storing", fmap dynTypeRep dyn)
232− ( \fs@FuzzState {..} ->
251+ ( \fs@FuzzState {..} ->
233− fs {stash = addToStash (NEL.toList dyn) stash}
252+ fs {stash = addToStash (NEL.toList dyn) stash}
236 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
255 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
237− argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
256+ -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
239 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
258 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
@@ -242,21 +261,28 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
243 (execute op func args)
262 (execute op func args)
244 [ Handler (\(e :: SomeAsyncException) -> throw e),
263 [ Handler (\(e :: SomeAsyncException) -> throw e),
265+ ( \(e :: SomeException) ->
245− Handler (\(e :: SomeException) -> throw . RoboservantException ServerCrashed (Just e) routeCount =<< get)
266+ throw . RoboservantException ServerCrashed (Just e) =<< get
248− (liftIO checker)
270+ (liftIO healthCheck)
249− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) routeCount =<< get)
271+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
251addToStash ::
273addToStash ::
252− [Dynamic] ->
274+ [(Dynamic, Int)] ->
255addToStash result stash =
277addToStash result stash =
257− ( \(Dynamic tr x) (Stash dict) ->
279+ ( \(Dynamic tr x, hashed) (Stash dict) ->
259− DM.insertWith renumber tr (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| [])) dict
284+ (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
@@ -265,11 +291,16 @@ addToStash result stash =
265 StashValue a ->
291 StashValue a ->
266 StashValue a ->
292 StashValue a ->
267 StashValue a
293 StashValue a
268− renumber (StashValue singleDyn) (StashValue l) = StashValue $ case NEL.toList singleDyn of
294+ renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
295+ | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
298+ ( case NEL.toList singleDyn of
269− [([Provenance tr _], dyn)] ->
299+ [([Provenance tr _], dyn)] ->
271− <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
300+ l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
272− _ -> error "should be impossible"
301+ _ -> error "should be impossible"
303+ (IntSet.union singleHash intSet)
274-- why isn't this in vinyl?
305-- why isn't this in vinyl?
275recordToList' ::
306recordToList' ::