@@ -2,53 +2,104 @@
22{-# LANGUAGE FlexibleContexts #-}
33{-# LANGUAGE GADTs #-}
44{-# LANGUAGE LambdaCase #-}
5−{-# LANGUAGE PolyKinds #-}
65{-# LANGUAGE RankNTypes #-}
76{-# LANGUAGE RecordWildCards #-}
87{-# LANGUAGE ScopedTypeVariables #-}
98{-# LANGUAGE TupleSections #-}
10−
9+{-# LANGUAGE TypeApplications #-}
1110{-# LANGUAGE TypeFamilies #-}
11+
1212{-# LANGUAGE UndecidableInstances #-}
1313
1414module Roboservant.Direct
15− ( fuzz',
16− Config (..),
17− -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
19− RoboservantException (..),
20− FuzzState (..),
21− FuzzOp (..),
22− FailureType (..),
23− Report (..),
15+ ( fuzz, Config(..)
16+
17+ -- TODO come up with something smarter than exporting all this, we should
18+ -- have some nice error-display functions
19+ , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
2420 )
2521where
2622
2723import Control.Exception.Lifted
28− ( Exception,
29− Handler (Handler),
30− SomeAsyncException,
31− SomeException,
32− catch,
33− catches,
34− handle,
35− throw,
36− )
24+ ( throw,
25+ handle,
26+ Handler(..),
27+ Exception,
28+ SomeException,
29+ SomeAsyncException,
30+ catch,
31+ catches )
32+import Control.Exception.Lifted
33+ ( throw,
34+ Handler(..),
35+ Exception,
36+ SomeException,
37+ SomeAsyncException,
38+ catch,
39+ catches )
40+import Control.Exception.Lifted
41+ ( Exception,
42+ Handler(..),
43+ SomeAsyncException,
44+ SomeException,
45+ catch,
46+ catches,
47+ handle,
48+ throw )
49+
3750import Control.Monad.State.Strict
38− ( MonadIO (..),
39− MonadState (get),
40− StateT (runStateT),
41− modify',
42− )
51+ ( MonadState, MonadIO, get, modify', liftIO, runStateT )
52+import Control.Monad.State.Strict
53+ ( MonadState, MonadIO, get, modify', liftIO, execStateT )
54+import Control.Monad.State.Strict
55+ ( MonadIO, MonadState, get, liftIO, modify', runStateT )
56+
57+import Control.Exception.Lifted(throw,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
58+import Control.Monad(void,replicateM,forM_)
59+import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,execStateT)
60+import Control.Monad.Trans.Control(MonadBaseControl)
61+import Data.Dependent.Map (DMap)
62+import Data.Dynamic (Dynamic(..), dynApply, dynTypeRep, fromDynamic)
63+import Data.List.NonEmpty (NonEmpty(..))
64+import Data.Map.Strict(Map)
65+import qualified Data.Set as Set
66+import Data.Maybe (mapMaybe)
67+import Data.Typeable (TypeRep)
68+import Servant ( Endpoints, Proxy(Proxy), Server, ServerError(..) )
69+import Servant ( Endpoints, Proxy(Proxy), Server, ServerError(..) )
70+
71+import System.Random ( StdGen, randomR, mkStdGen )
72+import System.Random ( StdGen, mkStdGen, randomR )
73+
74+import qualified Data.List.NonEmpty as NEL
75+import qualified Data.Map.Strict as Map
76+import Data.Time.Clock
77+import Data.Time.Clock
78+
79+import Data.List(sortOn)
80+import Control.Arrow(second)
81+import System.Timeout.Lifted(timeout)
82+import GHC.Generics ( (:*:)(..) )
83+import GHC.Generics ( (:*:)(..) )
84+
85+import qualified Data.Vinyl.Functor as V
86+import qualified Data.Vinyl.Functor as V
87+
88+import qualified Data.Vinyl.Curry as V
89+import qualified Data.Vinyl.Curry as V
90+
91+
92+
93+import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
94+import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
4395import Control.Monad.Trans.Control (MonadBaseControl)
4496import qualified Data.Dependent.Map as DM
45−import Data.Dynamic (Dynamic (..))
46−import qualified Data.IntSet as IntSet
97+import Data.Dynamic (Dynamic (..), dynTypeRep)
4798import Data.List.NonEmpty (NonEmpty (..))
4899import qualified Data.List.NonEmpty as NEL
49100import Data.Maybe (mapMaybe)
50101import qualified Data.Set as Set
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
102+import Data.Time.Clock
52103import qualified Data.Vinyl as V
53104import qualified Data.Vinyl.Curry as V
54105import qualified Data.Vinyl.Functor as V
@@ -56,26 +107,25 @@ import GHC.Generics ((:*:) (..))
56107import Roboservant.Types
57108 ( ApiOffset (..),
58109 Argument (..),
59− InteractionError(..),
110+ FlattenServer (..),
60111 Provenance (..),
61− ReifiedApi,
62112 ReifiedEndpoint (..),
63113 Stash (..),
64114 StashValue (..),
65− TypedF,
115+ ToReifiedApi (..),
116+ TypedF
66117 )
67−import Roboservant.Types.Config
68−
69−import System.Random (Random (randomR), StdGen, mkStdGen)
118+import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
119+import System.Random (StdGen, mkStdGen, randomR)
70120import qualified Type.Reflection as R
121+import qualified Data.IntSet as IntSet
71122
72123data RoboservantException
73124 = RoboservantException
74− { failureReason :: FailureType,
75− serverException :: Maybe SomeException,
76− fuzzState :: FuzzState
77− }
78− deriving (Show)
125+ { failureReason :: FailureType
126+ , serverException :: Maybe SomeException
127+ , fuzzState :: FuzzState
128+ } deriving (Show)
79129
80130instance Exception RoboservantException
81131
@@ -84,7 +134,7 @@ data FailureType
84134 | CheckerFailed
85135 | NoPossibleMoves
86136 | InsufficientCoverage Double
87− deriving (Show, Eq)
137+ deriving (Show,Eq)
88138
89139data FuzzOp
90140 = FuzzOp
@@ -93,6 +143,15 @@ data FuzzOp
93143 }
94144 deriving (Show, Eq)
95145
146+data Config
147+ = Config
148+ { seed :: [(Dynamic,Int)],
149+ maxRuntime :: Double, -- seconds to test for
150+ maxReps :: Integer,
151+ rngSeed :: Int,
152+ coverageThreshold :: Double
153+ }
154+
96155data FuzzState
97156 = FuzzState
98157 { path :: [FuzzOp],
@@ -105,7 +164,7 @@ data EndpointOption
105164 = forall as.
106165 (V.RecordToList as, V.RMap as) =>
107166 EndpointOption
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
167+ { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))),
109168 eoArgs :: V.Rec (TypedF StashValue) as
110169 }
111170
@@ -114,25 +173,19 @@ data StopReason
114173 | HitMaxIterations
115174 deriving (Show, Eq)
116175
117−data Report
118− = Report
119− { textual :: String,
120− rsException :: RoboservantException
121− }
176+data Report = Report
177+ { textual :: String
178+ , rsException :: RoboservantException}
122179 deriving (Show)
123180
124−
125−
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127−-- fuzzClient = undefined
128−
129−
130−
131−fuzz' ::
132− ReifiedApi ->
181+fuzz ::
182+ forall api.
183+ (FlattenServer api, ToReifiedApi (Endpoints api)) =>
184+ Server api ->
133185 Config ->
186+ IO () ->
134187 IO (Maybe Report)
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
188+fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
136189 let path = []
137190 stash = addToStash seed mempty
138191 currentRng = mkStdGen rngSeed
@@ -141,36 +194,39 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
141194 runStateT
142195 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
143196 FuzzState {..}
144− logInfo $ show stopreason
197+ print stopreason
145198 pure Nothing
146199 where
147200 -- something less terrible later
148201 formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
150− Report
151− (unlines [show failureType, show exception])
152− r
153− displayDiagnostics FuzzState {..} = liftIO $
154− logInfo $ unlines $
155− ["api endpoints covered"]
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
160− -- $ \_k v ->
161− -- (show . NEL.length $ getStashValue v))
162−
163− evaluateCoverage f@FuzzState {..}
202+ formatException r@(RoboservantException failureType exception _state) = Report
203+ (unlines [ show failureType , show exception ])
204+ r
205+
206+ displayDiagnostics FuzzState{..} = liftIO $ do
207+ putStrLn "api endpoints covered"
208+ mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
209+ putStrLn ""
210+ putStrLn "types in stash"
211+ DM.forWithKey_ (getStash stash) $ \k v ->
212+ print (NEL.length $ getStashValue v)
213+ -- mapM_
214+ -- (print . second NEL.length)
215+ -- (sortOn (show . fst) $ Map.toList $ getStash stash)
216+
217+ -- evaluateCoverage :: FuzzState -> m ()
218+ evaluateCoverage f@FuzzState{..}
164219 | coverage > coverageThreshold = pure ()
165220 | otherwise = do
166− displayDiagnostics f
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
168− where
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
170− totalRoutes = fromIntegral routeCount
171− coverage = hitRoutes / totalRoutes
172− untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
173− untilDone (0, _) _ = pure HitMaxIterations
221+ displayDiagnostics f
222+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
223+ where hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
224+ totalRoutes = fromIntegral routeCount
225+ coverage = hitRoutes / totalRoutes
226+
227+
228+ untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
229+ untilDone (0,_) _ = pure HitMaxIterations
174230 untilDone (n, deadline) action = do
175231 now <- liftIO getCurrentTime
176232 if now > deadline
@@ -178,13 +234,12 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
178234 else do
179235 _ <- action
180236 untilDone (n -1, deadline) action
181−
237+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
182238 routeCount = length reifiedApi
183− elementOrFail ::
184− (MonadState FuzzState m, MonadIO m) =>
185− [a] ->
186− m a
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
239+
240+ elementOrFail :: (MonadState FuzzState m, MonadIO m)
241+ => [a] -> m a
242+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
188243 elementOrFail l = do
189244 st <- get
190245 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
@@ -195,7 +250,7 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
195250 ( forall as.
196251 (V.RecordToList as, V.RMap as) =>
197252 FuzzOp ->
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
253+ V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))) ->
199254 V.Rec (TypedF V.Identity) as ->
200255 m r
201256 ) ->
@@ -236,53 +291,57 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
236291 execute ::
237292 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
238293 FuzzOp ->
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
294+ V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))) ->
240295 V.Rec (TypedF V.Identity) as ->
241296 m ()
242297 execute fuzzop func args = do
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
298+ (liftIO . print . (fuzzop,) . stash ) =<< get
299+ st <- get
300+ -- let showable = unlines $ ("args":map show argTypes)
301+ -- <> ["fuzzop"
302+ -- , show fuzzop
303+ -- -- ,"dyncall"
304+ -- -- ,show (dynTypeRep dyncall)
305+ -- ,"state"
306+ -- ]
307+ -- liftIO $ putStrLn showable
244308 liftIO (V.runcurry' func argVals) >>= \case
245− Left (e::InteractionError) ->
246− if fatalError e
247− then throw e
248− else pure ()
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
250− modify'
251− ( \fs@FuzzState {..} ->
252− fs {stash = addToStash (NEL.toList dyn) stash}
253− )
309+ -- parameterise this
310+ Left (serverError :: ServerError) ->
311+ case errHTTPCode serverError of
312+ 500 -> throw serverError
313+ _ -> do
314+ liftIO $ print ("ignoring non-500 error", serverError)
315+ Right (dyn :: NEL.NonEmpty (Dynamic,Int)) -> do
316+ -- liftIO $ print ("storing", fmap dynTypeRep dyn)
317+ modify' (\fs@FuzzState{..} ->
318+ fs { stash = addToStash (NEL.toList dyn) stash } )
254319 where
255320 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
321+ argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
257322 go ::
258323 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
259324 m ()
260325 go = withOp $ \op func args -> do
261− catches
262− (execute op func args)
263− [ Handler (\(e :: SomeAsyncException) -> throw e),
264− Handler
265− ( \(e :: SomeException) ->
266− throw . RoboservantException ServerCrashed (Just e) =<< get
267− )
326+ catches (execute op func args)
327+ [ Handler (\(e :: SomeAsyncException) -> throw e)
328+ , Handler (\(e :: SomeException) -> do
329+ displayDiagnostics =<< get
330+ throw . RoboservantException ServerCrashed (Just e) =<< get)
268331 ]
269− catch
270− (liftIO healthCheck)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
332+ catch (liftIO checker)
333+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
272334
273335addToStash ::
274− [(Dynamic, Int)] ->
336+ [(Dynamic,Int)] ->
275337 Stash ->
276338 Stash
277339addToStash result stash =
278340 foldr
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
341+ ( \(Dynamic tr x,hashed) (Stash dict) ->
280342 Stash $
281− DM.insertWith
282− renumber
283− tr
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
285− dict
343+ DM.insertWith renumber tr
344+ (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed)) dict
286345 )
287346 stash
288347 result
@@ -295,11 +354,12 @@ addToStash result stash =
295354 | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
296355 | otherwise =
297356 StashValue
298− ( case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
301− _ -> error "should be impossible"
302− )
357+ (case NEL.toList singleDyn of
358+ [([Provenance tr _], dyn)] ->
359+ l
360+(([Provenance tr (length (NEL.last l) + 1)], dyn))
361+ _ -> error "should be impossible")
362+
303363 (IntSet.union singleHash intSet)
304364
305365-- why isn't this in vinyl?