@@ -2,60 +2,53 @@
2{-# LANGUAGE FlexibleContexts #-}
2{-# LANGUAGE FlexibleContexts #-}
3{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
4{-# LANGUAGE LambdaCase #-}
4{-# LANGUAGE LambdaCase #-}
5+{-# LANGUAGE PolyKinds #-}
5{-# LANGUAGE RankNTypes #-}
6{-# LANGUAGE RankNTypes #-}
6{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE ScopedTypeVariables #-}
8{-# LANGUAGE ScopedTypeVariables #-}
8{-# LANGUAGE TupleSections #-}
9{-# LANGUAGE TupleSections #-}
9−{-# LANGUAGE TypeApplications #-}
10−{-# LANGUAGE TypeFamilies #-}
11+{-# LANGUAGE TypeFamilies #-}
12{-# LANGUAGE UndecidableInstances #-}
12{-# LANGUAGE UndecidableInstances #-}
14module Roboservant.Direct
14module Roboservant.Direct
15− ( fuzz, Config(..)
16+ Config (..),
17− -- TODO come up with something smarter than exporting all this, we should
17+ -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
18+ -- have some nice error-display functions
19− , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
19+ RoboservantException (..),
23−import Control.Exception.Lifted(throw,handle,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
27+import Control.Exception.Lifted
24−import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,runStateT)
28+ ( Exception,
25−import Control.Exception.Lifted(throw,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
29+ Handler (Handler),
26−import Control.Monad(void,replicateM,forM_)
30+ SomeAsyncException,
27−import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,execStateT)
37+import Control.Monad.State.Strict
28−import Control.Monad.Trans.Control(MonadBaseControl)
38+ ( MonadIO (..),
29−import Data.Dependent.Map (DMap)
39+ MonadState (get),
30−import Data.Dynamic (Dynamic(..), dynApply, dynTypeRep, fromDynamic)
40+ StateT (runStateT),
31−import Data.List.NonEmpty (NonEmpty(..))
41+ modify',
32−import Data.Map.Strict(Map)
42+ )
33−import qualified Data.Set as Set
34−import Data.Maybe (mapMaybe)
35−import Data.Typeable (TypeRep)
36−import Servant (Endpoints, Proxy (Proxy), Server, ServerError(..))
37−import System.Random(StdGen,randomR,mkStdGen)
38−import qualified Data.List.NonEmpty as NEL
39−import qualified Data.Map.Strict as Map
40−import Data.Time.Clock
41−import Data.List(sortOn)
42−import Control.Arrow(second)
43−import System.Timeout.Lifted(timeout)
44−import GHC.Generics ((:*:)(..))
45−import qualified Data.Vinyl.Functor as V
46−import qualified Data.Vinyl.Curry as V
49−import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
50−import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
51import Control.Monad.Trans.Control (MonadBaseControl)
43import Control.Monad.Trans.Control (MonadBaseControl)
52import qualified Data.Dependent.Map as DM
44import qualified Data.Dependent.Map as DM
53−import Data.Dynamic (Dynamic (..), dynTypeRep)
45+import Data.Dynamic (Dynamic (..))
46+import qualified Data.IntSet as IntSet
54import Data.List.NonEmpty (NonEmpty (..))
47import Data.List.NonEmpty (NonEmpty (..))
55import qualified Data.List.NonEmpty as NEL
48import qualified Data.List.NonEmpty as NEL
56import Data.Maybe (mapMaybe)
49import Data.Maybe (mapMaybe)
57import qualified Data.Set as Set
50import qualified Data.Set as Set
58−import Data.Time.Clock
51+import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
59import qualified Data.Vinyl as V
52import qualified Data.Vinyl as V
60import qualified Data.Vinyl.Curry as V
53import qualified Data.Vinyl.Curry as V
61import qualified Data.Vinyl.Functor as V
54import qualified Data.Vinyl.Functor as V
@@ -63,25 +56,26 @@ import GHC.Generics ((:*:) (..))
63import Roboservant.Types
56import Roboservant.Types
64 ( ApiOffset (..),
57 ( ApiOffset (..),
65 Argument (..),
58 Argument (..),
66− FlattenServer (..),
59+ InteractionError(..),
67 Provenance (..),
60 Provenance (..),
68 ReifiedEndpoint (..),
62 ReifiedEndpoint (..),
69 Stash (..),
63 Stash (..),
70 StashValue (..),
64 StashValue (..),
71− ToReifiedApi (..),
65+ TypedF,
74−import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
67+import Roboservant.Types.Config
75−import System.Random (StdGen, mkStdGen, randomR)
76−import qualified Type.Reflection as R
69+import System.Random (Random (randomR), StdGen, mkStdGen)
70+import qualified Type.Reflection as R
79data RoboservantException
72data RoboservantException
80 = RoboservantException
73 = RoboservantException
81− { failureReason :: FailureType
74+ { failureReason :: FailureType,
82− , serverException :: Maybe SomeException
75+ serverException :: Maybe SomeException,
83− , fuzzState :: FuzzState
76+ fuzzState :: FuzzState
84− } deriving (Show)
78+ deriving (Show)
86instance Exception RoboservantException
80instance Exception RoboservantException
@@ -90,7 +84,7 @@ data FailureType
90 | CheckerFailed
84 | CheckerFailed
91 | NoPossibleMoves
85 | NoPossibleMoves
92 | InsufficientCoverage Double
86 | InsufficientCoverage Double
93− deriving (Show,Eq)
87+ deriving (Show, Eq)
95data FuzzOp
89data FuzzOp
@@ -99,15 +93,6 @@ data FuzzOp
100 deriving (Show, Eq)
94 deriving (Show, Eq)
104− { seed :: [Dynamic],
105− maxRuntime :: Integer, -- seconds to test for
108− coverageThreshold :: Double
111data FuzzState
96data FuzzState
112 = FuzzState
97 = FuzzState
113 { path :: [FuzzOp],
98 { path :: [FuzzOp],
@@ -120,7 +105,7 @@ data EndpointOption
120 = forall as.
105 = forall as.
121 (V.RecordToList as, V.RMap as) =>
106 (V.RecordToList as, V.RMap as) =>
122 EndpointOption
107 EndpointOption
123− { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty Dynamic))),
108+ { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
124 eoArgs :: V.Rec (TypedF StashValue) as
109 eoArgs :: V.Rec (TypedF StashValue) as
@@ -129,60 +114,63 @@ data StopReason
129 | HitMaxIterations
114 | HitMaxIterations
130 deriving (Show, Eq)
115 deriving (Show, Eq)
132−data Report = Report
117+data Report
133− { textual :: String
119+ { textual :: String,
134− , rsException :: RoboservantException}
120+ rsException :: RoboservantException
135 deriving (Show)
122 deriving (Show)
126+-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127+-- fuzzClient = undefined
138− forall api.
132+ ReifiedApi ->
139− (FlattenServer api, ToReifiedApi (Endpoints api)) =>
141 Config ->
133 Config ->
143 IO (Maybe Report)
134 IO (Maybe Report)
144−fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
135+fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
145 let path = []
136 let path = []
146 stash = addToStash seed mempty
137 stash = addToStash seed mempty
147 currentRng = mkStdGen rngSeed
138 currentRng = mkStdGen rngSeed
148− deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
139+ deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
149 (stopreason, _fs) <-
140 (stopreason, _fs) <-
150 runStateT
141 runStateT
151 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
142 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
152 FuzzState {..}
143 FuzzState {..}
153− print stopreason
144+ logInfo $ show stopreason
154 pure Nothing
145 pure Nothing
156 -- something less terrible later
147 -- something less terrible later
157 formatException :: RoboservantException -> Report
148 formatException :: RoboservantException -> Report
158− formatException r@(RoboservantException failureType exception _state) = Report
149+ formatException r@(RoboservantException failureType exception _state) =
159− (unlines [ show failureType , show exception ])
151+ (unlines [show failureType, show exception])
153+ displayDiagnostics FuzzState {..} = liftIO $
155+ ["api endpoints covered"]
156+ <> (map show . Set.toList . Set.fromList $ map apiOffset path)
157+ <> ["", "types in stash"]
158+ <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
159+ -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
161+ -- (show . NEL.length $ getStashValue v))
162− displayDiagnostics FuzzState{..} = liftIO $ do
163− putStrLn "api endpoints covered"
164− mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
166− putStrLn "types in stash"
167− DM.forWithKey_ (getStash stash) $ \k v ->
168− print (NEL.length $ getStashValue v)
170− -- (print . second NEL.length)
171− -- (sortOn (show . fst) $ Map.toList $ getStash stash)
173− -- evaluateCoverage :: FuzzState -> m ()
174− evaluateCoverage f@FuzzState{..}
163+ evaluateCoverage f@FuzzState {..}
175 | coverage > coverageThreshold = pure ()
164 | coverage > coverageThreshold = pure ()
176 | otherwise = do
165 | otherwise = do
177− displayDiagnostics f
166+ displayDiagnostics f
178− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
167+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
179− where hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
169+ hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
180− totalRoutes = fromIntegral routeCount
170+ totalRoutes = fromIntegral routeCount
181− coverage = hitRoutes / totalRoutes
171+ coverage = hitRoutes / totalRoutes
184− untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
172+ untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
185− untilDone (0,_) _ = pure HitMaxIterations
173+ untilDone (0, _) _ = pure HitMaxIterations
186 untilDone (n, deadline) action = do
174 untilDone (n, deadline) action = do
187 now <- liftIO getCurrentTime
175 now <- liftIO getCurrentTime
188 if now > deadline
176 if now > deadline
@@ -190,12 +178,13 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
191 _ <- action
179 _ <- action
192 untilDone (n -1, deadline) action
180 untilDone (n -1, deadline) action
193− reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
194− routeCount = length reifiedApi
182+ routeCount = length reifiedApi
196− elementOrFail :: (MonadState FuzzState m, MonadIO m)
184+ (MonadState FuzzState m, MonadIO m) =>
197− => [a] -> m a
186+ m a
198− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
187+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
199 elementOrFail l = do
188 elementOrFail l = do
200 st <- get
189 st <- get
201 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
190 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
@@ -206,7 +195,7 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
206 ( forall as.
195 ( forall as.
207 (V.RecordToList as, V.RMap as) =>
196 (V.RecordToList as, V.RMap as) =>
208 FuzzOp ->
197 FuzzOp ->
209− V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
198+ V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
210 V.Rec (TypedF V.Identity) as ->
199 V.Rec (TypedF V.Identity) as ->
@@ -216,7 +205,7 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
216 (offset, EndpointOption {..}) <- elementOrFail . options =<< get
205 (offset, EndpointOption {..}) <- elementOrFail . options =<< get
218 V.rtraverse
207 V.rtraverse
219− ( \(tr :*: StashValue svs) ->
208+ ( \(tr :*: StashValue svs _) ->
220 elementOrFail $
209 elementOrFail $
222 (\i xy -> V.Const i :*: tr :*: xy)
211 (\i xy -> V.Const i :*: tr :*: xy)
@@ -247,56 +236,53 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
247 execute ::
236 execute ::
248 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
237 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
249 FuzzOp ->
238 FuzzOp ->
250− V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
239+ V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
251 V.Rec (TypedF V.Identity) as ->
240 V.Rec (TypedF V.Identity) as ->
253 execute fuzzop func args = do
242 execute fuzzop func args = do
254− (liftIO . print . (fuzzop,) . stash ) =<< get
243+ (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
256− let showable = unlines $ ("args":map show argTypes)
260− -- ,show (dynTypeRep dyncall)
263− -- liftIO $ putStrLn showable
264 liftIO (V.runcurry' func argVals) >>= \case
244 liftIO (V.runcurry' func argVals) >>= \case
265− -- parameterise this
245+ Left (e::InteractionError) ->
266− Left (serverError :: ServerError) ->
246+ if fatalError e
267− case errHTTPCode serverError of
247+ then throw e
268− 500 -> throw serverError
248+ else pure ()
270− liftIO $ print ("ignoring non-500 error", serverError)
271− Right (dyn :: NEL.NonEmpty Dynamic) -> do
249+ Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
272− -- liftIO $ print ("storing", fmap dynTypeRep dyn)
250+ modify'
273− modify' (\fs@FuzzState{..} ->
251+ ( \fs@FuzzState {..} ->
274− fs { stash = addToStash (NEL.toList dyn) stash } )
252+ fs {stash = addToStash (NEL.toList dyn) stash}
276 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
255 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
277− argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
256+ -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
279 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
258 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
281 go = withOp $ \op func args -> do
260 go = withOp $ \op func args -> do
282− catches (execute op func args)
262+ (execute op func args)
283− [ Handler (\(e :: SomeAsyncException) -> throw e)
263+ [ Handler (\(e :: SomeAsyncException) -> throw e),
284− , Handler (\(e :: SomeException) -> do
265+ ( \(e :: SomeException) ->
285− displayDiagnostics =<< get
286− throw . RoboservantException ServerCrashed (Just e) =<< get)
266+ throw . RoboservantException ServerCrashed (Just e) =<< get
288− catch (liftIO checker)
269+ catch
270+ (liftIO healthCheck)
289− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
271+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
291addToStash ::
273addToStash ::
292− [Dynamic] ->
274+ [(Dynamic, Int)] ->
295addToStash result stash =
277addToStash result stash =
297− ( \(Dynamic tr x) (Stash dict) ->
279+ ( \(Dynamic tr x, hashed) (Stash dict) ->
299− DM.insertWith renumber tr (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| [])) dict
284+ (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
@@ -305,11 +291,16 @@ addToStash result stash =
305 StashValue a ->
291 StashValue a ->
306 StashValue a ->
292 StashValue a ->
307 StashValue a
293 StashValue a
308− renumber (StashValue singleDyn) (StashValue l) = StashValue $ case NEL.toList singleDyn of
294+ renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
295+ | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
298+ ( case NEL.toList singleDyn of
309− [([Provenance tr _], dyn)] ->
299+ [([Provenance tr _], dyn)] ->
311− <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
300+ l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
312− _ -> error "should be impossible"
301+ _ -> error "should be impossible"
303+ (IntSet.union singleHash intSet)
314-- why isn't this in vinyl?
305-- why isn't this in vinyl?
315recordToList' ::
306recordToList' ::