@@ -2,104 +2,53 @@
2{-# LANGUAGE FlexibleContexts #-}
2{-# LANGUAGE FlexibleContexts #-}
3{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
4{-# LANGUAGE LambdaCase #-}
4{-# LANGUAGE LambdaCase #-}
5+{-# LANGUAGE PolyKinds #-}
5{-# LANGUAGE RankNTypes #-}
6{-# LANGUAGE RankNTypes #-}
6{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE ScopedTypeVariables #-}
8{-# LANGUAGE ScopedTypeVariables #-}
8{-# LANGUAGE TupleSections #-}
9{-# LANGUAGE TupleSections #-}
9−{-# LANGUAGE TypeApplications #-}
10−{-# LANGUAGE TypeFamilies #-}
11+{-# LANGUAGE TypeFamilies #-}
12{-# LANGUAGE UndecidableInstances #-}
12{-# LANGUAGE UndecidableInstances #-}
14module Roboservant.Direct
14module Roboservant.Direct
15− ( fuzz, Config(..)
16+ Config (..),
17− -- TODO come up with something smarter than exporting all this, we should
17+ -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
18+ -- have some nice error-display functions
19− , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
19+ RoboservantException (..),
23import Control.Exception.Lifted
27import Control.Exception.Lifted
27− Exception,
28+ ( Exception,
32−import Control.Exception.Lifted
34− Handler(..),
29+ Handler (Handler),
40−import Control.Exception.Lifted
43− SomeAsyncException,
30+ SomeAsyncException,
44− SomeException,
31+ SomeException,
50−import Control.Monad.State.Strict
51− ( MonadState, MonadIO, get, modify', liftIO, runStateT )
52−import Control.Monad.State.Strict
53− ( MonadState, MonadIO, get, modify', liftIO, execStateT )
54import Control.Monad.State.Strict
37import Control.Monad.State.Strict
55− ( MonadIO, MonadState, get, liftIO, modify', runStateT )
38+ ( MonadIO (..),
57−import Control.Exception.Lifted(throw,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
40+ StateT (runStateT),
58−import Control.Monad(void,replicateM,forM_)
41+ modify',
59−import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,execStateT)
42+ )
60−import Control.Monad.Trans.Control(MonadBaseControl)
61−import Data.Dependent.Map (DMap)
62−import Data.Dynamic (Dynamic(..), dynApply, dynTypeRep, fromDynamic)
63−import Data.List.NonEmpty (NonEmpty(..))
64−import Data.Map.Strict(Map)
65−import qualified Data.Set as Set
66−import Data.Maybe (mapMaybe)
67−import Data.Typeable (TypeRep)
68−import Servant ( Endpoints, Proxy(Proxy), Server, ServerError(..) )
69−import Servant ( Endpoints, Proxy(Proxy), Server, ServerError(..) )
71−import System.Random ( StdGen, randomR, mkStdGen )
72−import System.Random ( StdGen, mkStdGen, randomR )
74−import qualified Data.List.NonEmpty as NEL
75−import qualified Data.Map.Strict as Map
76−import Data.Time.Clock
77−import Data.Time.Clock
79−import Data.List(sortOn)
80−import Control.Arrow(second)
81−import System.Timeout.Lifted(timeout)
82−import GHC.Generics ( (:*:)(..) )
83−import GHC.Generics ( (:*:)(..) )
85−import qualified Data.Vinyl.Functor as V
86−import qualified Data.Vinyl.Functor as V
88−import qualified Data.Vinyl.Curry as V
89−import qualified Data.Vinyl.Curry as V
93−import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
94−import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
95import Control.Monad.Trans.Control (MonadBaseControl)
43import Control.Monad.Trans.Control (MonadBaseControl)
96import qualified Data.Dependent.Map as DM
44import qualified Data.Dependent.Map as DM
97−import Data.Dynamic (Dynamic (..), dynTypeRep)
45+import Data.Dynamic (Dynamic (..))
46+import qualified Data.IntSet as IntSet
98import Data.List.NonEmpty (NonEmpty (..))
47import Data.List.NonEmpty (NonEmpty (..))
99import qualified Data.List.NonEmpty as NEL
48import qualified Data.List.NonEmpty as NEL
100import Data.Maybe (mapMaybe)
49import Data.Maybe (mapMaybe)
101import qualified Data.Set as Set
50import qualified Data.Set as Set
102−import Data.Time.Clock
51+import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
103import qualified Data.Vinyl as V
52import qualified Data.Vinyl as V
104import qualified Data.Vinyl.Curry as V
53import qualified Data.Vinyl.Curry as V
105import qualified Data.Vinyl.Functor as V
54import qualified Data.Vinyl.Functor as V
@@ -107,25 +56,26 @@ import GHC.Generics ((:*:) (..))
107import Roboservant.Types
56import Roboservant.Types
108 ( ApiOffset (..),
57 ( ApiOffset (..),
109 Argument (..),
58 Argument (..),
110− FlattenServer (..),
59+ InteractionError(..),
111 Provenance (..),
60 Provenance (..),
112 ReifiedEndpoint (..),
62 ReifiedEndpoint (..),
113 Stash (..),
63 Stash (..),
114 StashValue (..),
64 StashValue (..),
115− ToReifiedApi (..),
65+ TypedF,
118−import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
67+import Roboservant.Types.Config
119−import System.Random (StdGen, mkStdGen, randomR)
69+import System.Random (Random (randomR), StdGen, mkStdGen)
120import qualified Type.Reflection as R
70import qualified Type.Reflection as R
121−import qualified Data.IntSet as IntSet
123data RoboservantException
72data RoboservantException
124 = RoboservantException
73 = RoboservantException
125− { failureReason :: FailureType
74+ { failureReason :: FailureType,
126− , serverException :: Maybe SomeException
75+ serverException :: Maybe SomeException,
127− , fuzzState :: FuzzState
76+ fuzzState :: FuzzState
128− } deriving (Show)
78+ deriving (Show)
130instance Exception RoboservantException
80instance Exception RoboservantException
@@ -134,7 +84,7 @@ data FailureType
134 | CheckerFailed
84 | CheckerFailed
135 | NoPossibleMoves
85 | NoPossibleMoves
136 | InsufficientCoverage Double
86 | InsufficientCoverage Double
137− deriving (Show,Eq)
87+ deriving (Show, Eq)
139data FuzzOp
89data FuzzOp
@@ -143,15 +93,6 @@ data FuzzOp
144 deriving (Show, Eq)
94 deriving (Show, Eq)
148− { seed :: [(Dynamic,Int)],
149− maxRuntime :: Double, -- seconds to test for
152− coverageThreshold :: Double
155data FuzzState
96data FuzzState
156 = FuzzState
97 = FuzzState
157 { path :: [FuzzOp],
98 { path :: [FuzzOp],
@@ -164,7 +105,7 @@ data EndpointOption
164 = forall as.
105 = forall as.
165 (V.RecordToList as, V.RMap as) =>
106 (V.RecordToList as, V.RMap as) =>
166 EndpointOption
107 EndpointOption
167− { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))),
108+ { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
168 eoArgs :: V.Rec (TypedF StashValue) as
109 eoArgs :: V.Rec (TypedF StashValue) as
@@ -173,19 +114,25 @@ data StopReason
173 | HitMaxIterations
114 | HitMaxIterations
174 deriving (Show, Eq)
115 deriving (Show, Eq)
176−data Report = Report
117+data Report
177− { textual :: String
119+ { textual :: String,
178− , rsException :: RoboservantException}
120+ rsException :: RoboservantException
179 deriving (Show)
122 deriving (Show)
126+-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127+-- fuzzClient = undefined
182− forall api.
132+ ReifiedApi ->
183− (FlattenServer api, ToReifiedApi (Endpoints api)) =>
185 Config ->
133 Config ->
187 IO (Maybe Report)
134 IO (Maybe Report)
188−fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
135+fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
189 let path = []
136 let path = []
190 stash = addToStash seed mempty
137 stash = addToStash seed mempty
191 currentRng = mkStdGen rngSeed
138 currentRng = mkStdGen rngSeed
@@ -194,39 +141,36 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
194 runStateT
141 runStateT
195 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
142 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
196 FuzzState {..}
143 FuzzState {..}
197− print stopreason
144+ logInfo $ show stopreason
198 pure Nothing
145 pure Nothing
200 -- something less terrible later
147 -- something less terrible later
201 formatException :: RoboservantException -> Report
148 formatException :: RoboservantException -> Report
202− formatException r@(RoboservantException failureType exception _state) = Report
149+ formatException r@(RoboservantException failureType exception _state) =
203− (unlines [ show failureType , show exception ])
151+ (unlines [show failureType, show exception])
206− displayDiagnostics FuzzState{..} = liftIO $ do
153+ displayDiagnostics FuzzState {..} = liftIO $
207− putStrLn "api endpoints covered"
155+ ["api endpoints covered"]
208− mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
156+ <> (map show . Set.toList . Set.fromList $ map apiOffset path)
210− putStrLn "types in stash"
157+ <> ["", "types in stash"]
211− DM.forWithKey_ (getStash stash) $ \k v ->
212− print (NEL.length $ getStashValue v)
158+ <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
214− -- (print . second NEL.length)
215− -- (sortOn (show . fst) $ Map.toList $ getStash stash)
159+ -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
217− -- evaluateCoverage :: FuzzState -> m ()
161+ -- (show . NEL.length $ getStashValue v))
218− evaluateCoverage f@FuzzState{..}
163+ evaluateCoverage f@FuzzState {..}
219 | coverage > coverageThreshold = pure ()
164 | coverage > coverageThreshold = pure ()
220 | otherwise = do
165 | otherwise = do
221− displayDiagnostics f
166+ displayDiagnostics f
222− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
167+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
223− where hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
169+ hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
224− totalRoutes = fromIntegral routeCount
170+ totalRoutes = fromIntegral routeCount
225− coverage = hitRoutes / totalRoutes
171+ coverage = hitRoutes / totalRoutes
228− untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
172+ untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
229− untilDone (0,_) _ = pure HitMaxIterations
173+ untilDone (0, _) _ = pure HitMaxIterations
230 untilDone (n, deadline) action = do
174 untilDone (n, deadline) action = do
231 now <- liftIO getCurrentTime
175 now <- liftIO getCurrentTime
232 if now > deadline
176 if now > deadline
@@ -234,12 +178,13 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
235 _ <- action
179 _ <- action
236 untilDone (n -1, deadline) action
180 untilDone (n -1, deadline) action
237− reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
238− routeCount = length reifiedApi
182+ routeCount = length reifiedApi
240− elementOrFail :: (MonadState FuzzState m, MonadIO m)
184+ (MonadState FuzzState m, MonadIO m) =>
241− => [a] -> m a
186+ m a
242− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
187+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
243 elementOrFail l = do
188 elementOrFail l = do
244 st <- get
189 st <- get
245 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
190 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
@@ -250,7 +195,7 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
250 ( forall as.
195 ( forall as.
251 (V.RecordToList as, V.RMap as) =>
196 (V.RecordToList as, V.RMap as) =>
252 FuzzOp ->
197 FuzzOp ->
253− V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))) ->
198+ V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
254 V.Rec (TypedF V.Identity) as ->
199 V.Rec (TypedF V.Identity) as ->
@@ -291,57 +236,53 @@ fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
291 execute ::
236 execute ::
292 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
237 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
293 FuzzOp ->
238 FuzzOp ->
294− V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))) ->
239+ V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
295 V.Rec (TypedF V.Identity) as ->
240 V.Rec (TypedF V.Identity) as ->
297 execute fuzzop func args = do
242 execute fuzzop func args = do
298− (liftIO . print . (fuzzop,) . stash ) =<< get
243+ (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
300− -- let showable = unlines $ ("args":map show argTypes)
304− -- -- ,show (dynTypeRep dyncall)
307− -- liftIO $ putStrLn showable
308 liftIO (V.runcurry' func argVals) >>= \case
244 liftIO (V.runcurry' func argVals) >>= \case
309− -- parameterise this
245+ Left (e::InteractionError) ->
310− Left (serverError :: ServerError) ->
246+ if fatalError e
311− case errHTTPCode serverError of
247+ then throw e
312− 500 -> throw serverError
248+ else pure ()
314− liftIO $ print ("ignoring non-500 error", serverError)
315− Right (dyn :: NEL.NonEmpty (Dynamic,Int)) -> do
249+ Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
316− -- liftIO $ print ("storing", fmap dynTypeRep dyn)
250+ modify'
317− modify' (\fs@FuzzState{..} ->
251+ ( \fs@FuzzState {..} ->
318− fs { stash = addToStash (NEL.toList dyn) stash } )
252+ fs {stash = addToStash (NEL.toList dyn) stash}
320 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
255 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
321− argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
256+ -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
323 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
258 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
325 go = withOp $ \op func args -> do
260 go = withOp $ \op func args -> do
326− catches (execute op func args)
262+ (execute op func args)
327− [ Handler (\(e :: SomeAsyncException) -> throw e)
263+ [ Handler (\(e :: SomeAsyncException) -> throw e),
328− , Handler (\(e :: SomeException) -> do
265+ ( \(e :: SomeException) ->
329− displayDiagnostics =<< get
330− throw . RoboservantException ServerCrashed (Just e) =<< get)
266+ throw . RoboservantException ServerCrashed (Just e) =<< get
332− catch (liftIO checker)
269+ catch
270+ (liftIO healthCheck)
333− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
271+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
335addToStash ::
273addToStash ::
336− [(Dynamic,Int)] ->
274+ [(Dynamic, Int)] ->
339addToStash result stash =
277addToStash result stash =
341− ( \(Dynamic tr x,hashed) (Stash dict) ->
279+ ( \(Dynamic tr x, hashed) (Stash dict) ->
343− DM.insertWith renumber tr
281+ DM.insertWith
344− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed)) dict
284+ (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
@@ -354,11 +295,11 @@ addToStash result stash =
354 | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
295 | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
355 | otherwise =
296 | otherwise =
356 StashValue
297 StashValue
357− (case NEL.toList singleDyn of
298+ ( case NEL.toList singleDyn of
358− [([Provenance tr _], dyn)] ->
299+ [([Provenance tr _], dyn)] ->
359− l <> pure (([Provenance tr (length (NEL.last l) + 1)], dyn))
300+ l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
360− _ -> error "should be impossible")
301+ _ -> error "should be impossible"
362 (IntSet.union singleHash intSet)
303 (IntSet.union singleHash intSet)
364-- why isn't this in vinyl?
305-- why isn't this in vinyl?