@@ -2,53 +2,60 @@
2{-# LANGUAGE FlexibleContexts #-}
2{-# LANGUAGE FlexibleContexts #-}
3{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
4{-# LANGUAGE LambdaCase #-}
4{-# LANGUAGE LambdaCase #-}
5−{-# LANGUAGE PolyKinds #-}
6{-# LANGUAGE RankNTypes #-}
5{-# LANGUAGE RankNTypes #-}
7{-# LANGUAGE RecordWildCards #-}
6{-# LANGUAGE RecordWildCards #-}
8{-# LANGUAGE ScopedTypeVariables #-}
7{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE TupleSections #-}
8{-# LANGUAGE TupleSections #-}
10−
9+{-# LANGUAGE TypeApplications #-}
11{-# LANGUAGE TypeFamilies #-}
10{-# LANGUAGE TypeFamilies #-}
12{-# LANGUAGE UndecidableInstances #-}
12{-# LANGUAGE UndecidableInstances #-}
14module Roboservant.Direct
14module Roboservant.Direct
16− Config (..),
15+ ( fuzz, Config(..)
17− -- TODO come up with something smarter than exporting all this, we should
17+ -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
18+ -- have some nice error-display functions
19− RoboservantException (..),
19+ , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
27−import Control.Exception.Lifted
23+import Control.Exception.Lifted(throw,handle,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
28− ( Exception,
24+import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,runStateT)
29− Handler (Handler),
25+import Control.Exception.Lifted(throw,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
30− SomeAsyncException,
26+import Control.Monad(void,replicateM,forM_)
37−import Control.Monad.State.Strict
27+import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,execStateT)
38− ( MonadIO (..),
28+import Control.Monad.Trans.Control(MonadBaseControl)
39− MonadState (get),
29+import Data.Dependent.Map (DMap)
40− StateT (runStateT),
30+import Data.Dynamic (Dynamic(..), dynApply, dynTypeRep, fromDynamic)
41− modify',
31+import Data.List.NonEmpty (NonEmpty(..))
42− )
32+import Data.Map.Strict(Map)
33+import qualified Data.Set as Set
34+import Data.Maybe (mapMaybe)
35+import Data.Typeable (TypeRep)
36+import Servant (Endpoints, Proxy (Proxy), Server, ServerError(..))
37+import System.Random(StdGen,randomR,mkStdGen)
38+import qualified Data.List.NonEmpty as NEL
39+import qualified Data.Map.Strict as Map
40+import Data.Time.Clock
41+import Data.List(sortOn)
42+import Control.Arrow(second)
43+import System.Timeout.Lifted(timeout)
44+import GHC.Generics ((:*:)(..))
45+import qualified Data.Vinyl.Functor as V
46+import qualified Data.Vinyl.Curry as V
49+import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
50+import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
43import Control.Monad.Trans.Control (MonadBaseControl)
51import Control.Monad.Trans.Control (MonadBaseControl)
44import qualified Data.Dependent.Map as DM
52import qualified Data.Dependent.Map as DM
45−import Data.Dynamic (Dynamic (..))
53+import Data.Dynamic (Dynamic (..), dynTypeRep)
46−import qualified Data.IntSet as IntSet
47import Data.List.NonEmpty (NonEmpty (..))
54import Data.List.NonEmpty (NonEmpty (..))
48import qualified Data.List.NonEmpty as NEL
55import qualified Data.List.NonEmpty as NEL
49import Data.Maybe (mapMaybe)
56import Data.Maybe (mapMaybe)
50import qualified Data.Set as Set
57import qualified Data.Set as Set
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
58+import Data.Time.Clock
52import qualified Data.Vinyl as V
59import qualified Data.Vinyl as V
53import qualified Data.Vinyl.Curry as V
60import qualified Data.Vinyl.Curry as V
54import qualified Data.Vinyl.Functor as V
61import qualified Data.Vinyl.Functor as V
@@ -56,26 +63,25 @@ import GHC.Generics ((:*:) (..))
56import Roboservant.Types
63import Roboservant.Types
57 ( ApiOffset (..),
64 ( ApiOffset (..),
58 Argument (..),
65 Argument (..),
59− InteractionError(..),
66+ FlattenServer (..),
60 Provenance (..),
67 Provenance (..),
62 ReifiedEndpoint (..),
68 ReifiedEndpoint (..),
63 Stash (..),
69 Stash (..),
64 StashValue (..),
70 StashValue (..),
67−import Roboservant.Types.Config
74+import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
69−import System.Random (Random (randomR), StdGen, mkStdGen)
75+import System.Random (StdGen, mkStdGen, randomR)
70import qualified Type.Reflection as R
76import qualified Type.Reflection as R
77+import qualified Data.IntSet as IntSet
72data RoboservantException
79data RoboservantException
73 = RoboservantException
80 = RoboservantException
74− { failureReason :: FailureType,
81+ { failureReason :: FailureType
75− serverException :: Maybe SomeException,
82+ , serverException :: Maybe SomeException
76− fuzzState :: FuzzState
83+ , fuzzState :: FuzzState
78− deriving (Show)
84+ } deriving (Show)
80instance Exception RoboservantException
86instance Exception RoboservantException
@@ -84,7 +90,7 @@ data FailureType
84 | CheckerFailed
90 | CheckerFailed
85 | NoPossibleMoves
91 | NoPossibleMoves
86 | InsufficientCoverage Double
92 | InsufficientCoverage Double
87− deriving (Show, Eq)
93+ deriving (Show,Eq)
89data FuzzOp
95data FuzzOp
@@ -93,6 +99,15 @@ data FuzzOp
94 deriving (Show, Eq)
100 deriving (Show, Eq)
104+ { seed :: [(Dynamic,Int)],
105+ maxRuntime :: Double, -- seconds to test for
108+ coverageThreshold :: Double
96data FuzzState
111data FuzzState
97 = FuzzState
112 = FuzzState
98 { path :: [FuzzOp],
113 { path :: [FuzzOp],
@@ -105,7 +120,7 @@ data EndpointOption
105 = forall as.
120 = forall as.
106 (V.RecordToList as, V.RMap as) =>
121 (V.RecordToList as, V.RMap as) =>
107 EndpointOption
122 EndpointOption
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
123+ { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))),
109 eoArgs :: V.Rec (TypedF StashValue) as
124 eoArgs :: V.Rec (TypedF StashValue) as
@@ -114,25 +129,19 @@ data StopReason
114 | HitMaxIterations
129 | HitMaxIterations
115 deriving (Show, Eq)
130 deriving (Show, Eq)
117−data Report
132+data Report = Report
119− { textual :: String,
133+ { textual :: String
120− rsException :: RoboservantException
134+ , rsException :: RoboservantException}
122 deriving (Show)
135 deriving (Show)
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127−-- fuzzClient = undefined
132− ReifiedApi ->
138+ forall api.
139+ (FlattenServer api, ToReifiedApi (Endpoints api)) =>
133 Config ->
141 Config ->
134 IO (Maybe Report)
143 IO (Maybe Report)
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
144+fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
136 let path = []
145 let path = []
137 stash = addToStash seed mempty
146 stash = addToStash seed mempty
138 currentRng = mkStdGen rngSeed
147 currentRng = mkStdGen rngSeed
@@ -141,36 +150,39 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
141 runStateT
150 runStateT
142 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
151 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
143 FuzzState {..}
152 FuzzState {..}
144− logInfo $ show stopreason
153+ print stopreason
145 pure Nothing
154 pure Nothing
147 -- something less terrible later
156 -- something less terrible later
148 formatException :: RoboservantException -> Report
157 formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
158+ formatException r@(RoboservantException failureType exception _state) = Report
151− (unlines [show failureType, show exception])
159+ (unlines [ show failureType , show exception ])
153− displayDiagnostics FuzzState {..} = liftIO $
162+ displayDiagnostics FuzzState{..} = liftIO $ do
155− ["api endpoints covered"]
163+ putStrLn "api endpoints covered"
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
164+ mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
166+ putStrLn "types in stash"
167+ DM.forWithKey_ (getStash stash) $ \k v ->
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
168+ print (NEL.length $ getStashValue v)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
169+ -- mapM_
160− -- $ \_k v ->
170+ -- (print . second NEL.length)
161− -- (show . NEL.length $ getStashValue v))
171+ -- (sortOn (show . fst) $ Map.toList $ getStash stash)
173+ -- evaluateCoverage :: FuzzState -> m ()
163− evaluateCoverage f@FuzzState {..}
174+ evaluateCoverage f@FuzzState{..}
164 | coverage > coverageThreshold = pure ()
175 | coverage > coverageThreshold = pure ()
165 | otherwise = do
176 | otherwise = do
166− displayDiagnostics f
177+ displayDiagnostics f
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
178+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
179+ where hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
170− totalRoutes = fromIntegral routeCount
180+ totalRoutes = fromIntegral routeCount
171− coverage = hitRoutes / totalRoutes
181+ coverage = hitRoutes / totalRoutes
172− untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
184+ untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
173− untilDone (0, _) _ = pure HitMaxIterations
185+ untilDone (0,_) _ = pure HitMaxIterations
174 untilDone (n, deadline) action = do
186 untilDone (n, deadline) action = do
175 now <- liftIO getCurrentTime
187 now <- liftIO getCurrentTime
176 if now > deadline
188 if now > deadline
@@ -178,13 +190,12 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
179 _ <- action
191 _ <- action
180 untilDone (n -1, deadline) action
192 untilDone (n -1, deadline) action
181−
193+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
182 routeCount = length reifiedApi
194 routeCount = length reifiedApi
183− elementOrFail ::
195+
184− (MonadState FuzzState m, MonadIO m) =>
196+ elementOrFail :: (MonadState FuzzState m, MonadIO m)
186− m a
197+ => [a] -> m a
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
198+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
188 elementOrFail l = do
199 elementOrFail l = do
189 st <- get
200 st <- get
190 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
201 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
@@ -195,7 +206,7 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
195 ( forall as.
206 ( forall as.
196 (V.RecordToList as, V.RMap as) =>
207 (V.RecordToList as, V.RMap as) =>
197 FuzzOp ->
208 FuzzOp ->
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
209+ V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))) ->
199 V.Rec (TypedF V.Identity) as ->
210 V.Rec (TypedF V.Identity) as ->
@@ -236,53 +247,57 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
236 execute ::
247 execute ::
237 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
248 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
238 FuzzOp ->
249 FuzzOp ->
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
250+ V.Curried as (IO (Either ServerError (NonEmpty (Dynamic,Int)))) ->
240 V.Rec (TypedF V.Identity) as ->
251 V.Rec (TypedF V.Identity) as ->
242 execute fuzzop func args = do
253 execute fuzzop func args = do
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
254+ (liftIO . print . (fuzzop,) . stash ) =<< get
256+ -- let showable = unlines $ ("args":map show argTypes)
260+ -- -- ,show (dynTypeRep dyncall)
263+ -- liftIO $ putStrLn showable
244 liftIO (V.runcurry' func argVals) >>= \case
264 liftIO (V.runcurry' func argVals) >>= \case
245− Left (e::InteractionError) ->
265+ -- parameterise this
246− if fatalError e
266+ Left (serverError :: ServerError) ->
247− then throw e
267+ case errHTTPCode serverError of
248− else pure ()
268+ 500 -> throw serverError
270+ liftIO $ print ("ignoring non-500 error", serverError)
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
271+ Right (dyn :: NEL.NonEmpty (Dynamic,Int)) -> do
250− modify'
272+ -- liftIO $ print ("storing", fmap dynTypeRep dyn)
251− ( \fs@FuzzState {..} ->
273+ modify' (\fs@FuzzState{..} ->
252− fs {stash = addToStash (NEL.toList dyn) stash}
274+ fs { stash = addToStash (NEL.toList dyn) stash } )
255 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
276 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
277+ argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
258 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
279 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
260 go = withOp $ \op func args -> do
281 go = withOp $ \op func args -> do
262− (execute op func args)
282+ catches (execute op func args)
263− [ Handler (\(e :: SomeAsyncException) -> throw e),
283+ [ Handler (\(e :: SomeAsyncException) -> throw e)
265− ( \(e :: SomeException) ->
284+ , Handler (\(e :: SomeException) -> do
285+ displayDiagnostics =<< get
266− throw . RoboservantException ServerCrashed (Just e) =<< get
286+ throw . RoboservantException ServerCrashed (Just e) =<< get)
269− catch
288+ catch (liftIO checker)
270− (liftIO healthCheck)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
289+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
273addToStash ::
291addToStash ::
274− [(Dynamic, Int)] ->
292+ [(Dynamic,Int)] ->
277addToStash result stash =
295addToStash result stash =
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
297+ ( \(Dynamic tr x,hashed) (Stash dict) ->
281− DM.insertWith
299+ DM.insertWith renumber tr
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
300+ (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed)) dict
@@ -295,11 +310,12 @@ addToStash result stash =
295 | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
310 | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
296 | otherwise =
311 | otherwise =
297 StashValue
312 StashValue
298− ( case NEL.toList singleDyn of
313+ (case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
314+ [([Provenance tr _], dyn)] ->
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
316+ <> pure (([Provenance tr (length (NEL.last l) + 1)], dyn))
301− _ -> error "should be impossible"
317+ _ -> error "should be impossible")
303 (IntSet.union singleHash intSet)
319 (IntSet.union singleHash intSet)
305-- why isn't this in vinyl?
321-- why isn't this in vinyl?