@@ -1,8 +1,5 @@
1{-# LANGUAGE AllowAmbiguousTypes #-}
1{-# LANGUAGE AllowAmbiguousTypes #-}
2−{-# LANGUAGE DataKinds #-}
3−{-# LANGUAGE DerivingStrategies #-}
4{-# LANGUAGE FlexibleContexts #-}
2{-# LANGUAGE FlexibleContexts #-}
5−{-# LANGUAGE FlexibleInstances #-}
6{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
7{-# LANGUAGE LambdaCase #-}
4{-# LANGUAGE LambdaCase #-}
8{-# LANGUAGE PolyKinds #-}
5{-# LANGUAGE PolyKinds #-}
@@ -10,71 +7,75 @@
10{-# LANGUAGE RecordWildCards #-}
7{-# LANGUAGE RecordWildCards #-}
11{-# LANGUAGE ScopedTypeVariables #-}
8{-# LANGUAGE ScopedTypeVariables #-}
12{-# LANGUAGE TupleSections #-}
9{-# LANGUAGE TupleSections #-}
13−{-# LANGUAGE TypeApplications #-}
14−{-# LANGUAGE TypeFamilies #-}
11+{-# LANGUAGE TypeFamilies #-}
16{-# LANGUAGE UndecidableInstances #-}
12{-# LANGUAGE UndecidableInstances #-}
18module Roboservant.Direct
14module Roboservant.Direct
19− ( fuzz, Config(..)
16+ Config (..),
21− -- TODO come up with something smarter than exporting all this, we should
17+ -- TODO come up with something smarter than exporting all this, we should
22− -- have some nice error-display functions
18+ -- have some nice error-display functions
23− , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
19+ RoboservantException (..),
27−import Control.Exception.Lifted(throw,handle,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
27+import Control.Exception.Lifted
28−import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,runStateT)
37+import Control.Monad.State.Strict
29−import Control.Exception.Lifted(throw,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
38+ ( MonadIO (..),
30−import Control.Monad(void,replicateM,forM_)
39+ MonadState (get),
31−import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,execStateT)
40+ StateT (runStateT),
32−import Control.Monad.Trans.Control(MonadBaseControl)
43+import Control.Monad.Trans.Control (MonadBaseControl)
33−import Data.Dependent.Map (DMap)
34−import Data.Dynamic (Dynamic(..), dynApply, dynTypeRep, fromDynamic)
35−import Data.List.NonEmpty (NonEmpty(..))
36−import Data.Map.Strict(Map)
37−import qualified Data.Set as Set
38−import Data.Maybe (mapMaybe)
39−import Data.Typeable (TypeRep)
40−import Servant (Endpoints, Proxy (Proxy), Server, ServerError(..))
41−import System.Random(StdGen,randomR,mkStdGen)
42−import qualified Data.List.NonEmpty as NEL
43−import qualified Data.Map.Strict as Map
44−import Data.Time.Clock
45−import Data.List(sortOn)
46−import Control.Arrow(second)
47−import System.Timeout.Lifted(timeout)
48−import GHC.Generics ((:*:)(..))
49−import qualified Data.Vinyl.Functor as V
50−import qualified Data.Vinyl.Curry as V
51import qualified Data.Dependent.Map as DM
44import qualified Data.Dependent.Map as DM
45+import Data.Dynamic (Dynamic (..))
52−import qualified Data.Vinyl as V
46+import qualified Data.IntSet as IntSet
47+import Data.List.NonEmpty (NonEmpty (..))
53import qualified Data.List.NonEmpty as NEL
48import qualified Data.List.NonEmpty as NEL
49+import Data.Maybe (mapMaybe)
54−import qualified Data.Map.Strict as Map
50+import qualified Data.Set as Set
55−import Data.Time.Clock
51+import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
56−import qualified Type.Reflection as R
52+import qualified Data.Vinyl as V
57−
53+import qualified Data.Vinyl.Curry as V
58−import Roboservant.Types.Breakdown
54+import qualified Data.Vinyl.Functor as V
55+import GHC.Generics ((:*:) (..))
59import Roboservant.Types
56import Roboservant.Types
60 ( ApiOffset (..),
57 ( ApiOffset (..),
61− FlattenServer (..),
58+ Argument (..),
59+ InteractionError(..),
62−-- ReifiedApi,
61+ ReifiedApi,
65− ReifiedEndpoint(..),
62+ ReifiedEndpoint (..),
66− Argument(..),
63+ Stash (..),
67+import Roboservant.Types.Config
69+import System.Random (Random (randomR), StdGen, mkStdGen)
70+import qualified Type.Reflection as R
72data RoboservantException
72data RoboservantException
73 = RoboservantException
73 = RoboservantException
74− { failureReason :: FailureType
74+ { failureReason :: FailureType,
75− , serverException :: Maybe SomeException
75+ serverException :: Maybe SomeException,
76− , fuzzState :: FuzzState
76+ fuzzState :: FuzzState
77− } deriving (Show)
78+ deriving (Show)
79instance Exception RoboservantException
80instance Exception RoboservantException
@@ -83,214 +84,228 @@ data FailureType
83 | CheckerFailed
84 | CheckerFailed
84 | NoPossibleMoves
85 | NoPossibleMoves
85 | InsufficientCoverage Double
86 | InsufficientCoverage Double
86− deriving (Show,Eq)
87+ deriving (Show, Eq)
89−data FuzzOp = FuzzOp
89+data FuzzOp
90− { apiOffset :: ApiOffset
91+ { apiOffset :: ApiOffset,
91− , provenance :: [Provenance]
92+ provenance :: [Provenance]
92− } deriving (Show,Eq)
94+ deriving (Show, Eq)
94−data Config
96+data FuzzState
97− , maxRuntime :: Integer -- seconds to test for
100− , coverageThreshold :: Double
103−data FuzzState = FuzzState
97+ = FuzzState
104− { path :: [FuzzOp]
98+ { path :: [FuzzOp],
105− , stash :: Stash
99+ stash :: Stash,
106− , currentRng :: StdGen
100+ currentRng :: StdGen
108− deriving Show
102+ deriving (Show)
110−data EndpointOption = forall as. (V.RecordToList as, V.RMap as) => EndpointOption
106+ (V.RecordToList as, V.RMap as) =>
111− { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
108+ { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
112− , eoArgs :: V.Rec (TypedF StashValue) as
109+ eoArgs :: V.Rec (TypedF StashValue) as
115data StopReason
112data StopReason
116 = TimedOut
113 = TimedOut
117 | HitMaxIterations
114 | HitMaxIterations
118− deriving (Show,Eq)
115+ deriving (Show, Eq)
120−data Report = Report
117+data Report
121− { textual :: String
119+ { textual :: String,
122− , rsException :: RoboservantException}
120+ rsException :: RoboservantException
123 deriving (Show)
122 deriving (Show)
125−fuzz :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api))
124+
127− -> Config
126+-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
128− -> IO ()
127+-- fuzzClient = undefined
129− -> IO (Maybe Report)
134+ IO (Maybe Report)
130−fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
135+fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
131 let path = []
136 let path = []
132 stash = addToStash seed mempty
137 stash = addToStash seed mempty
133 currentRng = mkStdGen rngSeed
138 currentRng = mkStdGen rngSeed
136− deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
139+ deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
137− (stopreason, fs ) <- runStateT
140+ (stopreason, _fs) <-
138− (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get)) FuzzState{..}
142+ (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
139− print stopreason
144+ logInfo $ show stopreason
140 pure Nothing
145 pure Nothing
142 -- something less terrible later
147 -- something less terrible later
143 formatException :: RoboservantException -> Report
148 formatException :: RoboservantException -> Report
144− formatException r@(RoboservantException failureType exception _state) = Report
149+ formatException r@(RoboservantException failureType exception _state) =
145− (unlines [ show failureType , show exception ])
151+ (unlines [show failureType, show exception])
148− displayDiagnostics FuzzState{..} = liftIO $ do
153+ displayDiagnostics FuzzState {..} = liftIO $
149− putStrLn "api endpoints covered"
155+ ["api endpoints covered"]
150− mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
156+ <> (map show . Set.toList . Set.fromList $ map apiOffset path)
152− putStrLn "types in stash"
157+ <> ["", "types in stash"]
153− DM.forWithKey_ (getStash stash) $ \k v ->
154− print (k,NEL.length $ getStashValue v)
158+ <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
156− -- (print . second NEL.length)
157− -- (sortOn (show . fst) $ Map.toList $ getStash stash)
159+ -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
159− -- evaluateCoverage :: FuzzState -> m ()
161+ -- (show . NEL.length $ getStashValue v))
160− evaluateCoverage f@FuzzState{..}
163+ evaluateCoverage f@FuzzState {..}
161 | coverage > coverageThreshold = pure ()
164 | coverage > coverageThreshold = pure ()
162 | otherwise = do
165 | otherwise = do
163− displayDiagnostics f
166+ displayDiagnostics f
164− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
167+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
165− where hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
169+ hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
166− totalRoutes = fromIntegral routeCount
170+ totalRoutes = fromIntegral routeCount
167− coverage = hitRoutes / totalRoutes
171+ coverage = hitRoutes / totalRoutes
170− untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
172+ untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
171− untilDone (0,_) _ = pure HitMaxIterations
173+ untilDone (0, _) _ = pure HitMaxIterations
172 untilDone (n, deadline) action = do
174 untilDone (n, deadline) action = do
173 now <- liftIO getCurrentTime
175 now <- liftIO getCurrentTime
174 if now > deadline
176 if now > deadline
175 then pure TimedOut
177 then pure TimedOut
177− action
179+ _ <- action
178− untilDone (n-1, deadline) action
180+ untilDone (n -1, deadline) action
180− reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
181 routeCount = length reifiedApi
182 routeCount = length reifiedApi
182−
183+ elementOrFail ::
183− elementOrFail :: (MonadState FuzzState m, MonadIO m)
184+ (MonadState FuzzState m, MonadIO m) =>
184− => [a] -> m a
186+ m a
185− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
187+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
186 elementOrFail l = do
188 elementOrFail l = do
187 st <- get
189 st <- get
188− let (index,newGen) = randomR (0, length l - 1) (currentRng st)
190+ let (index, newGen) = randomR (0, length l - 1) (currentRng st)
189− modify' $ \st' -> st' { currentRng = newGen }
191+ modify' $ \st' -> st' {currentRng = newGen}
190 pure (l !! index)
192 pure (l !! index)
192− withOp :: (MonadState FuzzState m, MonadIO m)
194+ (MonadState FuzzState m, MonadIO m) =>
193− => (forall as. (V.RecordToList as, V.RMap as)
196+ (V.RecordToList as, V.RMap as) =>
194− => FuzzOp
197+ FuzzOp ->
195− -> V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
198+ V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
196− -> V.Rec (TypedF V.Identity) as -> m r
199+ V.Rec (TypedF V.Identity) as ->
199 withOp callback = do
203 withOp callback = do
200 -- choose a call to make, from the endpoints with fillable arguments.
204 -- choose a call to make, from the endpoints with fillable arguments.
201− (offset, EndpointOption{..}) <- elementOrFail . options =<< get
205+ (offset, EndpointOption {..}) <- elementOrFail . options =<< get
202− r <- V.rtraverse
207+ V.rtraverse
203− (\(tr :*: StashValue svs) -> elementOrFail $
208+ ( \(tr :*: StashValue svs _) ->
204− zipWith (\i xy -> V.Const i :*: tr :*: xy)
211+ (\i xy -> V.Const i :*: tr :*: xy)
206− (NEL.toList svs)
213+ (NEL.toList svs)
209− let pathSegment = FuzzOp offset $
216+ let pathSegment =
210− recordToList'
218+ recordToList'
211− (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
219+ (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
223+ (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
214− (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
216− modify' (\f -> f { path = path f <> [pathSegment] })
225+ modify' (\f -> f {path = path f <> [pathSegment]})
217 callback pathSegment eoCall argValues
226 callback pathSegment eoCall argValues
219 options :: FuzzState -> [(ApiOffset, EndpointOption)]
228 options :: FuzzState -> [(ApiOffset, EndpointOption)]
220− options FuzzState{..} =
229+ options FuzzState {..} =
222− ( \(offset, ReifiedEndpoint{..}) -> do
231+ ( \(offset, ReifiedEndpoint {..}) -> do
223− args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
232+ args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
224− pure (offset, EndpointOption reEndpointFunc args)
233+ pure (offset, EndpointOption reEndpointFunc args)
226 reifiedApi
235 reifiedApi
228− execute :: (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as)
237+ (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
229− => FuzzOp
238+ FuzzOp ->
230− -> V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
239+ V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
231− -> V.Rec (TypedF V.Identity) as
240+ V.Rec (TypedF V.Identity) as ->
233 execute fuzzop func args = do
242 execute fuzzop func args = do
234− liftIO $ print fuzzop
243+ (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
236− let showable = unlines $ ("args":map show argTypes)
240− -- ,show (dynTypeRep dyncall)
243− liftIO $ putStrLn showable
244 liftIO (V.runcurry' func argVals) >>= \case
244 liftIO (V.runcurry' func argVals) >>= \case
245− -- parameterise this
245+ Left (e::InteractionError) ->
246− Left (serverError :: ServerError) ->
246+ if fatalError e
247− case errHTTPCode serverError of
247+ then throw e
248− 500 -> throw serverError
248+ else pure ()
250− liftIO $ print ("ignoring non-500 error" , serverError)
251− Right (dyn :: NEL.NonEmpty Dynamic) -> do
249+ Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
252− liftIO $ print ("storing", fmap dynTypeRep dyn)
250+ modify'
253− modify' (\fs@FuzzState{..} ->
251+ ( \fs@FuzzState {..} ->
254− fs { stash = addToStash (NEL.toList dyn) stash } )
252+ fs {stash = addToStash (NEL.toList dyn) stash}
256 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
255 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
257− argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
256+ -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
259− go :: (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m)
258+ (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
261 go = withOp $ \op func args -> do
260 go = withOp $ \op func args -> do
262− catches (execute op func args)
262+ (execute op func args)
263− [ Handler (\(e :: SomeAsyncException) -> throw e)
263+ [ Handler (\(e :: SomeAsyncException) -> throw e),
264− , Handler (\(e :: SomeException) -> do
265+ ( \(e :: SomeException) ->
265− displayDiagnostics =<< get
266− throw . RoboservantException ServerCrashed (Just e) =<< get)
266+ throw . RoboservantException ServerCrashed (Just e) =<< get
268− catch (liftIO checker)
269+ catch
270+ (liftIO healthCheck)
269− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
271+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
271−addToStash :: [Dynamic]
273+addToStash ::
272− -> Stash
274+ [(Dynamic, Int)] ->
274addToStash result stash =
277addToStash result stash =
275− foldr (\(Dynamic tr x) (Stash dict) -> Stash $
279+ ( \(Dynamic tr x, hashed) (Stash dict) ->
276− DM.insertWith renumber tr (StashValue (([Provenance (R.SomeTypeRep tr) 0],x):|[])) dict
284+ (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
281− renumber :: StashValue a
291+ StashValue a ->
282− -> StashValue a
292+ StashValue a ->
283− -> StashValue a
293+ StashValue a
284− renumber (StashValue singleDyn) (StashValue l) = StashValue $ case NEL.toList singleDyn of
294+ renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
295+ | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
298+ ( case NEL.toList singleDyn of
285− [([Provenance tr _], dyn)] -> l
299+ [([Provenance tr _], dyn)] ->
286− <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
300+ l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
287− _ -> error "should be impossible"
301+ _ -> error "should be impossible"
303+ (IntSet.union singleHash intSet)
290-- why isn't this in vinyl?
305-- why isn't this in vinyl?
291−recordToList'
306+recordToList' ::
292− :: (V.RecordToList as, V.RMap as)
307+ (V.RecordToList as, V.RMap as) =>
293− => (forall x. f x -> a)
308+ (forall x. f x -> a) ->
294− -> V.Rec f as
309+ V.Rec f as ->
296recordToList' f = V.recordToList . V.rmap (V.Const . f)
311recordToList' f = V.recordToList . V.rmap (V.Const . f)