@@ -1,5 +1,8 @@
1{-# LANGUAGE AllowAmbiguousTypes #-}
1{-# LANGUAGE AllowAmbiguousTypes #-}
2+{-# LANGUAGE DataKinds #-}
3+{-# LANGUAGE DerivingStrategies #-}
2{-# LANGUAGE FlexibleContexts #-}
4{-# LANGUAGE FlexibleContexts #-}
5+{-# LANGUAGE FlexibleInstances #-}
3{-# LANGUAGE GADTs #-}
6{-# LANGUAGE GADTs #-}
4{-# LANGUAGE LambdaCase #-}
7{-# LANGUAGE LambdaCase #-}
5{-# LANGUAGE PolyKinds #-}
8{-# LANGUAGE PolyKinds #-}
@@ -7,75 +10,71 @@
7{-# LANGUAGE RecordWildCards #-}
10{-# LANGUAGE RecordWildCards #-}
8{-# LANGUAGE ScopedTypeVariables #-}
11{-# LANGUAGE ScopedTypeVariables #-}
9{-# LANGUAGE TupleSections #-}
12{-# LANGUAGE TupleSections #-}
10−
13+{-# LANGUAGE TypeApplications #-}
11{-# LANGUAGE TypeFamilies #-}
14{-# LANGUAGE TypeFamilies #-}
12{-# LANGUAGE UndecidableInstances #-}
16{-# LANGUAGE UndecidableInstances #-}
14module Roboservant.Direct
18module Roboservant.Direct
16− Config (..),
19+ ( fuzz, Config(..)
17− -- TODO come up with something smarter than exporting all this, we should
21+ -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
22+ -- have some nice error-display functions
19− RoboservantException (..),
23+ , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
27−import Control.Exception.Lifted
27+import Control.Exception.Lifted(throw,handle,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
37−import Control.Monad.State.Strict
28+import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,runStateT)
38− ( MonadIO (..),
29+import Control.Exception.Lifted(throw,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
39− MonadState (get),
30+import Control.Monad(void,replicateM,forM_)
40− StateT (runStateT),
31+import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,execStateT)
43−import Control.Monad.Trans.Control (MonadBaseControl)
32+import Control.Monad.Trans.Control(MonadBaseControl)
44−import qualified Data.Dependent.Map as DM
33+import Data.Dependent.Map (DMap)
45−import Data.Dynamic (Dynamic (..))
34+import Data.Dynamic (Dynamic(..), dynApply, dynTypeRep, fromDynamic)
46−import qualified Data.IntSet as IntSet
47−import Data.List.NonEmpty (NonEmpty (..))
35+import Data.List.NonEmpty (NonEmpty(..))
48−import qualified Data.List.NonEmpty as NEL
36+import Data.Map.Strict(Map)
49−import Data.Maybe (mapMaybe)
50import qualified Data.Set as Set
37import qualified Data.Set as Set
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
38+import Data.Maybe (mapMaybe)
52−import qualified Data.Vinyl as V
39+import Data.Typeable (TypeRep)
53−import qualified Data.Vinyl.Curry as V
40+import Servant (Endpoints, Proxy (Proxy), Server, ServerError(..))
41+import System.Random(StdGen,randomR,mkStdGen)
42+import qualified Data.List.NonEmpty as NEL
43+import qualified Data.Map.Strict as Map
44+import Data.Time.Clock
45+import Data.List(sortOn)
46+import Control.Arrow(second)
47+import System.Timeout.Lifted(timeout)
48+import GHC.Generics ((:*:)(..))
54import qualified Data.Vinyl.Functor as V
49import qualified Data.Vinyl.Functor as V
55−import GHC.Generics ((:*:) (..))
50+import qualified Data.Vinyl.Curry as V
51+import qualified Data.Dependent.Map as DM
52+import qualified Data.Vinyl as V
53+import qualified Data.List.NonEmpty as NEL
54+import qualified Data.Map.Strict as Map
55+import Data.Time.Clock
56+import qualified Type.Reflection as R
58+import Roboservant.Types.Breakdown
56import Roboservant.Types
59import Roboservant.Types
57 ( ApiOffset (..),
60 ( ApiOffset (..),
58− Argument (..),
61+ FlattenServer (..),
59− InteractionError(..),
61− ReifiedApi,
62+-- ReifiedApi,
62− ReifiedEndpoint (..),
65+ ReifiedEndpoint(..),
63− Stash (..),
66+ Argument(..),
67−import Roboservant.Types.Config
69−import System.Random (Random (randomR), StdGen, mkStdGen)
70−import qualified Type.Reflection as R
72data RoboservantException
72data RoboservantException
73 = RoboservantException
73 = RoboservantException
74− { failureReason :: FailureType,
74+ { failureReason :: FailureType
75− serverException :: Maybe SomeException,
75+ , serverException :: Maybe SomeException
76− fuzzState :: FuzzState
76+ , fuzzState :: FuzzState
78− deriving (Show)
77+ } deriving (Show)
80instance Exception RoboservantException
79instance Exception RoboservantException
@@ -84,228 +83,214 @@ data FailureType
84 | CheckerFailed
83 | CheckerFailed
85 | NoPossibleMoves
84 | NoPossibleMoves
86 | InsufficientCoverage Double
85 | InsufficientCoverage Double
87− deriving (Show, Eq)
86+ deriving (Show,Eq)
91− { apiOffset :: ApiOffset,
92− provenance :: [Provenance]
100− currentRng :: StdGen
106− (V.RecordToList as, V.RMap as) =>
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
109− eoArgs :: V.Rec (TypedF StashValue) as
112−data StopReason
89+data FuzzOp = FuzzOp
113− = TimedOut
90+ { apiOffset :: ApiOffset
114− | HitMaxIterations
91+ , provenance :: [Provenance]
119− { textual :: String,
120− rsException :: RoboservantException
122− deriving (Show)
92+ } deriving (Show,Eq)
97+ , maxRuntime :: Integer -- seconds to test for
100+ , coverageThreshold :: Double
103+data FuzzState = FuzzState
106+ , currentRng :: StdGen
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
110+data EndpointOption = forall as. (V.RecordToList as, V.RMap as) => EndpointOption
127−-- fuzzClient = undefined
111+ { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
112+ , eoArgs :: V.Rec (TypedF StashValue) as
122+ , rsException :: RoboservantException}
131−fuzz' ::
125+fuzz :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api))
132− ReifiedApi ->
126+ => Server api
133− Config ->
127+ -> Config
134− IO (Maybe Report)
129+ -> IO (Maybe Report)
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
130+fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
136 let path = []
131 let path = []
137 stash = addToStash seed mempty
132 stash = addToStash seed mempty
138 currentRng = mkStdGen rngSeed
133 currentRng = mkStdGen rngSeed
135+ print ("Initial seed", seed, stash)
139− deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
136+ deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
140− (stopreason, _fs) <-
141− runStateT
137+ (stopreason, fs ) <- runStateT
142− (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
138+ (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get)) FuzzState{..}
144− logInfo $ show stopreason
139+ print stopreason
145 pure Nothing
140 pure Nothing
147 -- something less terrible later
142 -- something less terrible later
148 formatException :: RoboservantException -> Report
143 formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
144+ formatException r@(RoboservantException failureType exception _state) = Report
151− (unlines [show failureType, show exception])
145+ (unlines [ show failureType , show exception ])
153− displayDiagnostics FuzzState {..} = liftIO $
148+ displayDiagnostics FuzzState{..} = liftIO $ do
155− ["api endpoints covered"]
149+ putStrLn "api endpoints covered"
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
150+ mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
152+ putStrLn "types in stash"
153+ DM.forWithKey_ (getStash stash) $ \k v ->
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
154+ print (NEL.length $ getStashValue v)
156+ -- (print . second NEL.length)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
157+ -- (sortOn (show . fst) $ Map.toList $ getStash stash)
161− -- (show . NEL.length $ getStashValue v))
159+ -- evaluateCoverage :: FuzzState -> m ()
163− evaluateCoverage f@FuzzState {..}
160+ evaluateCoverage f@FuzzState{..}
164 | coverage > coverageThreshold = pure ()
161 | coverage > coverageThreshold = pure ()
165 | otherwise = do
162 | otherwise = do
166− displayDiagnostics f
163+ displayDiagnostics f
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
164+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
165+ where hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
170− totalRoutes = fromIntegral routeCount
166+ totalRoutes = fromIntegral routeCount
171− coverage = hitRoutes / totalRoutes
167+ coverage = hitRoutes / totalRoutes
172− untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
170+ untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
173− untilDone (0, _) _ = pure HitMaxIterations
171+ untilDone (0,_) _ = pure HitMaxIterations
174 untilDone (n, deadline) action = do
172 untilDone (n, deadline) action = do
175 now <- liftIO getCurrentTime
173 now <- liftIO getCurrentTime
176 if now > deadline
174 if now > deadline
177 then pure TimedOut
175 then pure TimedOut
179− _ <- action
177+ action
180− untilDone (n -1, deadline) action
178+ untilDone (n-1, deadline) action
180+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
182 routeCount = length reifiedApi
181 routeCount = length reifiedApi
183− elementOrFail ::
182+
184− (MonadState FuzzState m, MonadIO m) =>
183+ elementOrFail :: (MonadState FuzzState m, MonadIO m)
186− m a
184+ => [a] -> m a
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
185+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
188 elementOrFail l = do
186 elementOrFail l = do
189 st <- get
187 st <- get
190− let (index, newGen) = randomR (0, length l - 1) (currentRng st)
188+ let (index,newGen) = randomR (0, length l - 1) (currentRng st)
191− modify' $ \st' -> st' {currentRng = newGen}
189+ modify' $ \st' -> st' { currentRng = newGen }
192 pure (l !! index)
190 pure (l !! index)
194− (MonadState FuzzState m, MonadIO m) =>
192+ withOp :: (MonadState FuzzState m, MonadIO m)
196− (V.RecordToList as, V.RMap as) =>
193+ => (forall as. (V.RecordToList as, V.RMap as)
197− FuzzOp ->
194+ => FuzzOp
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
195+ -> V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
199− V.Rec (TypedF V.Identity) as ->
196+ -> V.Rec (TypedF V.Identity) as -> m r
203 withOp callback = do
199 withOp callback = do
204 -- choose a call to make, from the endpoints with fillable arguments.
200 -- choose a call to make, from the endpoints with fillable arguments.
205− (offset, EndpointOption {..}) <- elementOrFail . options =<< get
201+ (offset, EndpointOption{..}) <- elementOrFail . options =<< get
207− V.rtraverse
202+ r <- V.rtraverse
208− ( \(tr :*: StashValue svs _) ->
203+ (\(tr :*: StashValue svs) -> elementOrFail $
211− (\i xy -> V.Const i :*: tr :*: xy)
204+ zipWith (\i xy -> V.Const i :*: tr :*: xy)
213− (NEL.toList svs)
206+ (NEL.toList svs)
216− let pathSegment =
209+ let pathSegment = FuzzOp offset $
218− recordToList'
210+ recordToList'
219− (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
211+ (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
223− (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
214+ (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
225− modify' (\f -> f {path = path f <> [pathSegment]})
216+ modify' (\f -> f { path = path f <> [pathSegment] })
226 callback pathSegment eoCall argValues
217 callback pathSegment eoCall argValues
228 options :: FuzzState -> [(ApiOffset, EndpointOption)]
219 options :: FuzzState -> [(ApiOffset, EndpointOption)]
229− options FuzzState {..} =
220+ options FuzzState{..} =
231− ( \(offset, ReifiedEndpoint {..}) -> do
222+ ( \(offset, ReifiedEndpoint{..}) -> do
232− args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
223+ args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
233− pure (offset, EndpointOption reEndpointFunc args)
224+ pure (offset, EndpointOption reEndpointFunc args)
235 reifiedApi
226 reifiedApi
237− (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
228+ execute :: (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as)
238− FuzzOp ->
229+ => FuzzOp
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
230+ -> V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
240− V.Rec (TypedF V.Identity) as ->
231+ -> V.Rec (TypedF V.Identity) as
242 execute fuzzop func args = do
233 execute fuzzop func args = do
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
234+ (liftIO . print . (fuzzop,) . stash ) =<< get
236+ let showable = unlines $ ("args":map show argTypes)
240+ -- ,show (dynTypeRep dyncall)
243+ -- liftIO $ putStrLn showable
244 liftIO (V.runcurry' func argVals) >>= \case
244 liftIO (V.runcurry' func argVals) >>= \case
245− Left (e::InteractionError) ->
245+ -- parameterise this
246− if fatalError e
246+ Left (serverError :: ServerError) ->
247− then throw e
247+ case errHTTPCode serverError of
248− else pure ()
248+ 500 -> throw serverError
250+ liftIO $ print ("ignoring non-500 error" , serverError)
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
251+ Right (dyn :: NEL.NonEmpty Dynamic) -> do
250− modify'
252+ -- liftIO $ print ("storing", fmap dynTypeRep dyn)
251− ( \fs@FuzzState {..} ->
253+ modify' (\fs@FuzzState{..} ->
252− fs {stash = addToStash (NEL.toList dyn) stash}
254+ fs { stash = addToStash (NEL.toList dyn) stash } )
255 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
257+ argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
258− (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
259+ go :: (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m)
260 go = withOp $ \op func args -> do
261 go = withOp $ \op func args -> do
262− (execute op func args)
262+ catches (execute op func args)
263− [ Handler (\(e :: SomeAsyncException) -> throw e),
263+ [ Handler (\(e :: SomeAsyncException) -> throw e)
265− ( \(e :: SomeException) ->
264+ , Handler (\(e :: SomeException) -> do
265+ displayDiagnostics =<< get
266− throw . RoboservantException ServerCrashed (Just e) =<< get
266+ throw . RoboservantException ServerCrashed (Just e) =<< get)
269− catch
268+ catch (liftIO checker)
270− (liftIO healthCheck)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
269+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
273−addToStash ::
271+addToStash :: [Dynamic]
274− [(Dynamic, Int)] ->
272+ -> Stash
277addToStash result stash =
274addToStash result stash =
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
275+ foldr (\(Dynamic tr x) (Stash dict) -> Stash $
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
276+ DM.insertWith renumber tr (StashValue (([Provenance (R.SomeTypeRep tr) 0],x):|[])) dict
291− StashValue a ->
281+ renumber :: StashValue a
292− StashValue a ->
282+ -> StashValue a
293− StashValue a
283+ -> StashValue a
294− renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
284+ renumber (StashValue singleDyn) (StashValue l) = StashValue $ case NEL.toList singleDyn of
295− | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
298− ( case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
285+ [([Provenance tr _], dyn)] -> l
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
286+ <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
301− _ -> error "should be impossible"
287+ _ -> error "should be impossible"
303− (IntSet.union singleHash intSet)
305-- why isn't this in vinyl?
290-- why isn't this in vinyl?
306−recordToList' ::
291+recordToList'
307− (V.RecordToList as, V.RMap as) =>
292+ :: (V.RecordToList as, V.RMap as)
308− (forall x. f x -> a) ->
293+ => (forall x. f x -> a)
309− V.Rec f as ->
294+ -> V.Rec f as
311recordToList' f = V.recordToList . V.rmap (V.Const . f)
296recordToList' f = V.recordToList . V.rmap (V.Const . f)