@@ -2,53 +2,33 @@
22{-# LANGUAGE FlexibleContexts #-}
33{-# LANGUAGE GADTs #-}
44{-# LANGUAGE LambdaCase #-}
5−{-# LANGUAGE PolyKinds #-}
65{-# LANGUAGE RankNTypes #-}
76{-# LANGUAGE RecordWildCards #-}
87{-# LANGUAGE ScopedTypeVariables #-}
98{-# LANGUAGE TupleSections #-}
10−
11−{-# LANGUAGE TypeFamilies #-}
12−{-# LANGUAGE UndecidableInstances #-}
9+{-# LANGUAGE TypeApplications #-}
1310
1411module Roboservant.Direct
15− ( fuzz',
12+ ( fuzz,
1613 Config (..),
1714 -- TODO come up with something smarter than exporting all this, we should
1815 -- have some nice error-display functions
1916 RoboservantException (..),
2017 FuzzState (..),
2118 FuzzOp (..),
22− FailureType (..),
23− Report (..),
2419 )
2520where
2621
27−import Control.Exception.Lifted
28− ( Exception,
29− Handler (Handler),
30− SomeAsyncException,
31− SomeException,
32− catch,
33− catches,
34− handle,
35− throw,
36− )
37−import Control.Monad.State.Strict
38− ( MonadIO (..),
39− MonadState (get),
40− StateT (runStateT),
41− modify',
42− )
22+import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
23+import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
4324import Control.Monad.Trans.Control (MonadBaseControl)
4425import qualified Data.Dependent.Map as DM
45−import Data.Dynamic (Dynamic (..))
46−import qualified Data.IntSet as IntSet
26+import Data.Dynamic (Dynamic (..), dynTypeRep)
4727import Data.List.NonEmpty (NonEmpty (..))
4828import qualified Data.List.NonEmpty as NEL
4929import Data.Maybe (mapMaybe)
5030import qualified Data.Set as Set
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
31+import Data.Time.Clock
5232import qualified Data.Vinyl as V
5333import qualified Data.Vinyl.Curry as V
5434import qualified Data.Vinyl.Functor as V
@@ -56,34 +36,30 @@ import GHC.Generics ((:*:) (..))
5636import Roboservant.Types
5737 ( ApiOffset (..),
5838 Argument (..),
59− InteractionError(..),
39+ FlattenServer (..),
6040 Provenance (..),
61− ReifiedApi,
6241 ReifiedEndpoint (..),
6342 Stash (..),
6443 StashValue (..),
44+ ToReifiedApi (..),
6545 TypedF,
6646 )
67−import Roboservant.Types.Config
68−
69−import System.Random (Random (randomR), StdGen, mkStdGen)
47+import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
48+import System.Random (StdGen, mkStdGen, randomR)
7049import qualified Type.Reflection as R
7150
7251data RoboservantException
73− = RoboservantException
74− { failureReason :: FailureType,
75− serverException :: Maybe SomeException,
76− fuzzState :: FuzzState
77− }
52+ = RoboservantException FailureType (Maybe SomeException) Int FuzzState
7853 deriving (Show)
7954
55+-- we believe in nussink, lebowski
8056instance Exception RoboservantException
8157
8258data FailureType
8359 = ServerCrashed
8460 | CheckerFailed
8561 | NoPossibleMoves
86− | InsufficientCoverage Double
62+ | InsufficientCoverage
8763 deriving (Show, Eq)
8864
8965data FuzzOp
@@ -93,6 +69,15 @@ data FuzzOp
9369 }
9470 deriving (Show, Eq)
9571
72+data Config
73+ = Config
74+ { seed :: [Dynamic],
75+ maxRuntime :: Integer, -- seconds to test for
76+ maxReps :: Integer,
77+ rngSeed :: Int,
78+ coverageThreshold :: Double
79+ }
80+
9681data FuzzState
9782 = FuzzState
9883 { path :: [FuzzOp],
@@ -105,7 +90,7 @@ data EndpointOption
10590 = forall as.
10691 (V.RecordToList as, V.RMap as) =>
10792 EndpointOption
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
93+ { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty Dynamic))),
10994 eoArgs :: V.Rec (TypedF StashValue) as
11095 }
11196
@@ -116,59 +101,40 @@ data StopReason
116101
117102data Report
118103 = Report
119− { textual :: String,
120− rsException :: RoboservantException
121− }
122− deriving (Show)
123−
124−
125−
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127−-- fuzzClient = undefined
128−
129−
104+ {textual :: String}
105+ deriving (Show, Eq)
130106
131−fuzz' ::
132− ReifiedApi ->
107+fuzz ::
108+ forall api.
109+ (FlattenServer api, ToReifiedApi (Endpoints api)) =>
110+ Server api ->
133111 Config ->
112+ IO () ->
134113 IO (Maybe Report)
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
114+fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
136115 let path = []
137116 stash = addToStash seed mempty
138117 currentRng = mkStdGen rngSeed
139− deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
118+ deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
140119 (stopreason, _fs) <-
141120 runStateT
142121 (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
143122 FuzzState {..}
144− logInfo $ show stopreason
123+ print stopreason
145124 pure Nothing
146125 where
147126 -- something less terrible later
148127 formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
150− Report
151− (unlines [show failureType, show exception])
152− r
153− displayDiagnostics FuzzState {..} = liftIO $
154− logInfo $ unlines $
155− ["api endpoints covered"]
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
160− -- $ \_k v ->
161− -- (show . NEL.length $ getStashValue v))
162−
128+ formatException = Report . show
129+ -- evaluateCoverage :: FuzzState -> m ()
163130 evaluateCoverage f@FuzzState {..}
164− | coverage > coverageThreshold = pure ()
165− | otherwise = do
166− displayDiagnostics f
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
131+ | hitRoutes / totalRoutes > coverageThreshold = do
132+ liftIO $ print ("passed coverage", hitRoutes, totalRoutes)
133+ pure ()
134+ | otherwise = throw $ RoboservantException InsufficientCoverage Nothing routeCount f
168135 where
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
170− totalRoutes = fromIntegral routeCount
171− coverage = hitRoutes / totalRoutes
136+ hitRoutes = (fromIntegral . Set.size . Set.fromList $ map apiOffset path)
137+ totalRoutes = (fromIntegral routeCount)
172138 untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
173139 untilDone (0, _) _ = pure HitMaxIterations
174140 untilDone (n, deadline) action = do
@@ -178,13 +144,13 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
178144 else do
179145 _ <- action
180146 untilDone (n -1, deadline) action
181−
147+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
182148 routeCount = length reifiedApi
183149 elementOrFail ::
184150 (MonadState FuzzState m, MonadIO m) =>
185151 [a] ->
186152 m a
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
153+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing routeCount =<< get
188154 elementOrFail l = do
189155 st <- get
190156 let (index, newGen) = randomR (0, length l - 1) (currentRng st)
@@ -195,7 +161,7 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
195161 ( forall as.
196162 (V.RecordToList as, V.RMap as) =>
197163 FuzzOp ->
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
164+ V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
199165 V.Rec (TypedF V.Identity) as ->
200166 m r
201167 ) ->
@@ -205,7 +171,7 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
205171 (offset, EndpointOption {..}) <- elementOrFail . options =<< get
206172 r <-
207173 V.rtraverse
208− ( \(tr :*: StashValue svs _) ->
174+ ( \(tr :*: StashValue svs) ->
209175 elementOrFail $
210176 zipWith
211177 (\i xy -> V.Const i :*: tr :*: xy)
@@ -236,24 +202,39 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
236202 execute ::
237203 (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
238204 FuzzOp ->
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
205+ V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
240206 V.Rec (TypedF V.Identity) as ->
241207 m ()
242208 execute fuzzop func args = do
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
209+ liftIO $ print fuzzop
210+ st <- get
211+ let showable =
212+ unlines $
213+ ("args" : map show argTypes)
214+ <> [ "fuzzop",
215+ show fuzzop,
216+ -- ,"dyncall"
217+ -- ,show (dynTypeRep dyncall)
218+ "state",
219+ show st
220+ ]
221+ liftIO $ putStrLn showable
244222 liftIO (V.runcurry' func argVals) >>= \case
245− Left (e::InteractionError) ->
246− if fatalError e
247− then throw e
248− else pure ()
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
223+ -- parameterise this
224+ Left (serverError :: ServerError) ->
225+ case errHTTPCode serverError of
226+ 500 -> throw serverError
227+ _ -> do
228+ liftIO $ print ("ignoring non-500 error", serverError)
229+ Right (dyn :: NEL.NonEmpty Dynamic) -> do
230+ liftIO $ print ("storing", fmap dynTypeRep dyn)
250231 modify'
251− ( \fs@FuzzState {..} ->
252− fs {stash = addToStash (NEL.toList dyn) stash}
253− )
232+ ( \fs@FuzzState {..} ->
233+ fs {stash = addToStash (NEL.toList dyn) stash}
234+ )
254235 where
255236 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
237+ argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
257238 go ::
258239 (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
259240 m ()
@@ -261,28 +242,21 @@ fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
261242 catches
262243 (execute op func args)
263244 [ Handler (\(e :: SomeAsyncException) -> throw e),
264− Handler
265− ( \(e :: SomeException) ->
266− throw . RoboservantException ServerCrashed (Just e) =<< get
267− )
245+ Handler (\(e :: SomeException) -> throw . RoboservantException ServerCrashed (Just e) routeCount =<< get)
268246 ]
269247 catch
270− (liftIO healthCheck)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
248+ (liftIO checker)
249+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) routeCount =<< get)
272250
273251addToStash ::
274− [(Dynamic, Int)] ->
252+ [Dynamic] ->
275253 Stash ->
276254 Stash
277255addToStash result stash =
278256 foldr
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
257+ ( \(Dynamic tr x) (Stash dict) ->
280258 Stash $
281− DM.insertWith
282− renumber
283− tr
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
285− dict
259+ DM.insertWith renumber tr (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| [])) dict
286260 )
287261 stash
288262 result
@@ -291,16 +265,11 @@ addToStash result stash =
291265 StashValue a ->
292266 StashValue a ->
293267 StashValue a
294− renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
295− | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
296− | otherwise =
297− StashValue
298− ( case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
301− _ -> error "should be impossible"
302− )
303− (IntSet.union singleHash intSet)
268+ renumber (StashValue singleDyn) (StashValue l) = StashValue $ case NEL.toList singleDyn of
269+ [([Provenance tr _], dyn)] ->
270+ l
271+ <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
272+ _ -> error "should be impossible"
304273
305274-- why isn't this in vinyl?
306275recordToList' ::