@@ -2,53 +2,33 @@
2{-# LANGUAGE FlexibleContexts #-}
2{-# LANGUAGE FlexibleContexts #-}
3{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
4{-# LANGUAGE LambdaCase #-}
4{-# LANGUAGE LambdaCase #-}
5−{-# LANGUAGE PolyKinds #-}
6{-# LANGUAGE RankNTypes #-}
5{-# LANGUAGE RankNTypes #-}
7{-# LANGUAGE RecordWildCards #-}
6{-# LANGUAGE RecordWildCards #-}
8{-# LANGUAGE ScopedTypeVariables #-}
7{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE TupleSections #-}
8{-# LANGUAGE TupleSections #-}
10−
9+{-# LANGUAGE TypeApplications #-}
11−{-# LANGUAGE TypeFamilies #-}
12−{-# LANGUAGE UndecidableInstances #-}
14module Roboservant.Direct
11module Roboservant.Direct
16 Config (..),
13 Config (..),
17 -- TODO come up with something smarter than exporting all this, we should
14 -- TODO come up with something smarter than exporting all this, we should
18 -- have some nice error-display functions
15 -- have some nice error-display functions
19 RoboservantException (..),
16 RoboservantException (..),
20 FuzzState (..),
17 FuzzState (..),
21 FuzzOp (..),
18 FuzzOp (..),
27−import Control.Exception.Lifted
22+import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
37−import Control.Monad.State.Strict
23+import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
43import Control.Monad.Trans.Control (MonadBaseControl)
24import Control.Monad.Trans.Control (MonadBaseControl)
44import qualified Data.Dependent.Map as DM
25import qualified Data.Dependent.Map as DM
45−import Data.Dynamic (Dynamic (..))
26+import Data.Dynamic (Dynamic (..), dynTypeRep)
46−import qualified Data.IntSet as IntSet
47import Data.List.NonEmpty (NonEmpty (..))
27import Data.List.NonEmpty (NonEmpty (..))
48import qualified Data.List.NonEmpty as NEL
28import qualified Data.List.NonEmpty as NEL
49import Data.Maybe (mapMaybe)
29import Data.Maybe (mapMaybe)
50import qualified Data.Set as Set
30import qualified Data.Set as Set
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
31+import Data.Time.Clock
52import qualified Data.Vinyl as V
32import qualified Data.Vinyl as V
53import qualified Data.Vinyl.Curry as V
33import qualified Data.Vinyl.Curry as V
54import qualified Data.Vinyl.Functor as V
34import qualified Data.Vinyl.Functor as V
@@ -56,34 +36,30 @@ import GHC.Generics ((:*:) (..))
56import Roboservant.Types
36import Roboservant.Types
57 ( ApiOffset (..),
37 ( ApiOffset (..),
58 Argument (..),
38 Argument (..),
59− InteractionError(..),
39+ FlattenServer (..),
60 Provenance (..),
40 Provenance (..),
62 ReifiedEndpoint (..),
41 ReifiedEndpoint (..),
63 Stash (..),
42 Stash (..),
64 StashValue (..),
43 StashValue (..),
67−import Roboservant.Types.Config
47+import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
69−import System.Random (Random (randomR), StdGen, mkStdGen)
48+import System.Random (StdGen, mkStdGen, randomR)
70import qualified Type.Reflection as R
49import qualified Type.Reflection as R
72data RoboservantException
51data RoboservantException
73− = RoboservantException
52+ = RoboservantException FailureType (Maybe SomeException) Int FuzzState
74− { failureReason :: FailureType,
75− serverException :: Maybe SomeException,
76− fuzzState :: FuzzState
78 deriving (Show)
53 deriving (Show)
55+-- we believe in nussink, lebowski
80instance Exception RoboservantException
56instance Exception RoboservantException
82data FailureType
58data FailureType
83 = ServerCrashed
59 = ServerCrashed
84 | CheckerFailed
60 | CheckerFailed
85 | NoPossibleMoves
61 | NoPossibleMoves
86− | InsufficientCoverage Double
62+ | InsufficientCoverage
87 deriving (Show, Eq)
63 deriving (Show, Eq)
89data FuzzOp
65data FuzzOp
@@ -93,6 +69,15 @@ data FuzzOp
94 deriving (Show, Eq)
70 deriving (Show, Eq)
75+ maxRuntime :: Integer, -- seconds to test for
78+ coverageThreshold :: Double
96data FuzzState
81data FuzzState
97 = FuzzState
82 = FuzzState
98 { path :: [FuzzOp],
83 { path :: [FuzzOp],
@@ -105,7 +90,7 @@ data EndpointOption
105 = forall as.
90 = forall as.
106 (V.RecordToList as, V.RMap as) =>
91 (V.RecordToList as, V.RMap as) =>
107 EndpointOption
92 EndpointOption
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
93+ { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty Dynamic))),
109 eoArgs :: V.Rec (TypedF StashValue) as
94 eoArgs :: V.Rec (TypedF StashValue) as
@@ -116,59 +101,40 @@ data StopReason
117data Report
102data Report
119− { textual :: String,
104+ {textual :: String}
120− rsException :: RoboservantException
122− deriving (Show)
105+ deriving (Show, Eq)
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127−-- fuzzClient = undefined
132− ReifiedApi ->
108+ forall api.
109+ (FlattenServer api, ToReifiedApi (Endpoints api)) =>
133 Config ->
111 Config ->
134 IO (Maybe Report)
113 IO (Maybe Report)
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
114+fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
136 let path = []
115 let path = []
137 stash = addToStash seed mempty
116 stash = addToStash seed mempty
138 currentRng = mkStdGen rngSeed
117 currentRng = mkStdGen rngSeed
139− deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
118+ deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
140 (stopreason, _fs) <-
119 (stopreason, _fs) <-
141 runStateT
120 runStateT
142 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
121 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
143 FuzzState {..}
122 FuzzState {..}
144− logInfo $ show stopreason
123+ print stopreason
145 pure Nothing
124 pure Nothing
147 -- something less terrible later
126 -- something less terrible later
148 formatException :: RoboservantException -> Report
127 formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
128+ formatException = Report . show
150− Report
129+ -- evaluateCoverage :: FuzzState -> m ()
151− (unlines [show failureType, show exception])
153− displayDiagnostics FuzzState {..} = liftIO $
155− ["api endpoints covered"]
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
161− -- (show . NEL.length $ getStashValue v))
163 evaluateCoverage f@FuzzState {..}
130 evaluateCoverage f@FuzzState {..}
164− | coverage > coverageThreshold = pure ()
131+ | hitRoutes / totalRoutes > coverageThreshold = do
165− | otherwise = do
132+ liftIO $ print ("passed coverage", hitRoutes, totalRoutes)
166− displayDiagnostics f
133+ pure ()
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
134+ | otherwise = throw $ RoboservantException InsufficientCoverage Nothing routeCount f
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
136+ hitRoutes = (fromIntegral . Set.size . Set.fromList $ map apiOffset path)
170− totalRoutes = fromIntegral routeCount
137+ totalRoutes = (fromIntegral routeCount)
171− coverage = hitRoutes / totalRoutes
172 untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
138 untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
173 untilDone (0, _) _ = pure HitMaxIterations
139 untilDone (0, _) _ = pure HitMaxIterations
174 untilDone (n, deadline) action = do
140 untilDone (n, deadline) action = do
@@ -178,13 +144,13 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
179 _ <- action
145 _ <- action
180 untilDone (n -1, deadline) action
146 untilDone (n -1, deadline) action
181−
147+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
182 routeCount = length reifiedApi
148 routeCount = length reifiedApi
183 elementOrFail ::
149 elementOrFail ::
184 (MonadState FuzzState m, MonadIO m) =>
150 (MonadState FuzzState m, MonadIO m) =>
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
153+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing routeCount =<< get
188 elementOrFail l = do
154 elementOrFail l = do
189 st <- get
155 st <- get
190 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
156 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
@@ -195,7 +161,7 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
195 ( forall as.
161 ( forall as.
196 (V.RecordToList as, V.RMap as) =>
162 (V.RecordToList as, V.RMap as) =>
197 FuzzOp ->
163 FuzzOp ->
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
164+ V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
199 V.Rec (TypedF V.Identity) as ->
165 V.Rec (TypedF V.Identity) as ->
@@ -205,7 +171,7 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
205 (offset, EndpointOption {..}) <- elementOrFail . options =<< get
171 (offset, EndpointOption {..}) <- elementOrFail . options =<< get
207 V.rtraverse
173 V.rtraverse
208− ( \(tr :*: StashValue svs _) ->
174+ ( \(tr :*: StashValue svs) ->
209 elementOrFail $
175 elementOrFail $
211 (\i xy -> V.Const i :*: tr :*: xy)
177 (\i xy -> V.Const i :*: tr :*: xy)
@@ -236,24 +202,39 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
236 execute ::
202 execute ::
237 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
203 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
238 FuzzOp ->
204 FuzzOp ->
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
205+ V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
240 V.Rec (TypedF V.Identity) as ->
206 V.Rec (TypedF V.Identity) as ->
242 execute fuzzop func args = do
208 execute fuzzop func args = do
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
209+ liftIO $ print fuzzop
213+ ("args" : map show argTypes)
217+ -- ,show (dynTypeRep dyncall)
221+ liftIO $ putStrLn showable
244 liftIO (V.runcurry' func argVals) >>= \case
222 liftIO (V.runcurry' func argVals) >>= \case
245− Left (e::InteractionError) ->
223+ -- parameterise this
246− if fatalError e
224+ Left (serverError :: ServerError) ->
247− then throw e
225+ case errHTTPCode serverError of
248− else pure ()
226+ 500 -> throw serverError
228+ liftIO $ print ("ignoring non-500 error", serverError)
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
229+ Right (dyn :: NEL.NonEmpty Dynamic) -> do
230+ liftIO $ print ("storing", fmap dynTypeRep dyn)
251− ( \fs@FuzzState {..} ->
232+ ( \fs@FuzzState {..} ->
252− fs {stash = addToStash (NEL.toList dyn) stash}
233+ fs {stash = addToStash (NEL.toList dyn) stash}
255 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
236 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
237+ argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
258 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
239 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
@@ -261,28 +242,21 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
262 (execute op func args)
243 (execute op func args)
263 [ Handler (\(e :: SomeAsyncException) -> throw e),
244 [ Handler (\(e :: SomeAsyncException) -> throw e),
265− ( \(e :: SomeException) ->
266− throw . RoboservantException ServerCrashed (Just e) =<< get
245+ Handler (\(e :: SomeException) -> throw . RoboservantException ServerCrashed (Just e) routeCount =<< get)
270− (liftIO healthCheck)
248+ (liftIO checker)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
249+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) routeCount =<< get)
273addToStash ::
251addToStash ::
274− [(Dynamic, Int)] ->
252+ [Dynamic] ->
277addToStash result stash =
255addToStash result stash =
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
257+ ( \(Dynamic tr x) (Stash dict) ->
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
259+ DM.insertWith renumber tr (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| [])) dict
@@ -291,16 +265,11 @@ addToStash result stash =
291 StashValue a ->
265 StashValue a ->
292 StashValue a ->
266 StashValue a ->
293 StashValue a
267 StashValue a
294− renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
268+ renumber (StashValue singleDyn) (StashValue l) = StashValue $ case NEL.toList singleDyn of
295− | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
298− ( case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
269+ [([Provenance tr _], dyn)] ->
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
271+ <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
301− _ -> error "should be impossible"
272+ _ -> error "should be impossible"
303− (IntSet.union singleHash intSet)
305-- why isn't this in vinyl?
274-- why isn't this in vinyl?
306recordToList' ::
275recordToList' ::