@@ -1,11 +1,7 @@
1{-# LANGUAGE AllowAmbiguousTypes #-}
1{-# LANGUAGE AllowAmbiguousTypes #-}
2−{-# LANGUAGE DataKinds #-}
3−{-# LANGUAGE DerivingStrategies #-}
4{-# LANGUAGE FlexibleContexts #-}
2{-# LANGUAGE FlexibleContexts #-}
5−{-# LANGUAGE FlexibleInstances #-}
6{-# LANGUAGE GADTs #-}
3{-# LANGUAGE GADTs #-}
7{-# LANGUAGE LambdaCase #-}
4{-# LANGUAGE LambdaCase #-}
8−{-# LANGUAGE PolyKinds #-}
9{-# LANGUAGE RankNTypes #-}
5{-# LANGUAGE RankNTypes #-}
10{-# LANGUAGE RecordWildCards #-}
6{-# LANGUAGE RecordWildCards #-}
11{-# LANGUAGE ScopedTypeVariables #-}
7{-# LANGUAGE ScopedTypeVariables #-}
@@ -48,25 +44,36 @@ import System.Timeout.Lifted(timeout)
48import GHC.Generics ((:*:)(..))
44import GHC.Generics ((:*:)(..))
49import qualified Data.Vinyl.Functor as V
45import qualified Data.Vinyl.Functor as V
50import qualified Data.Vinyl.Curry as V
46import qualified Data.Vinyl.Curry as V
49+import Control.Exception.Lifted (Exception, Handler (..), SomeAsyncException, SomeException, catch, catches, handle, throw)
50+import Control.Monad.State.Strict (MonadIO, MonadState, get, liftIO, modify', runStateT)
51+import Control.Monad.Trans.Control (MonadBaseControl)
51import qualified Data.Dependent.Map as DM
52import qualified Data.Dependent.Map as DM
52−import qualified Data.Vinyl as V
53+import Data.Dynamic (Dynamic (..), dynTypeRep)
54+import Data.List.NonEmpty (NonEmpty (..))
53import qualified Data.List.NonEmpty as NEL
55import qualified Data.List.NonEmpty as NEL
56+import Data.Maybe (mapMaybe)
54−import qualified Data.Map.Strict as Map
57+import qualified Data.Set as Set
55import Data.Time.Clock
58import Data.Time.Clock
56−import qualified Type.Reflection as R
59+import qualified Data.Vinyl as V
57−
60+import qualified Data.Vinyl.Curry as V
58−import Roboservant.Types.Breakdown
61+import qualified Data.Vinyl.Functor as V
62+import GHC.Generics ((:*:) (..))
59import Roboservant.Types
63import Roboservant.Types
60 ( ApiOffset (..),
64 ( ApiOffset (..),
61 FlattenServer (..),
66 FlattenServer (..),
62−-- ReifiedApi,
67+ Provenance (..),
68+ ReifiedEndpoint (..),
63 ToReifiedApi (..),
71 ToReifiedApi (..),
74+import Servant (Endpoints, Proxy (Proxy), Server, ServerError (..))
75+import System.Random (StdGen, mkStdGen, randomR)
76+import qualified Type.Reflection as R
72data RoboservantException
79data RoboservantException
@@ -85,57 +92,64 @@ data FailureType
85 | InsufficientCoverage Double
92 | InsufficientCoverage Double
86 deriving (Show,Eq)
93 deriving (Show,Eq)
89−data FuzzOp = FuzzOp
95+data FuzzOp
90− { apiOffset :: ApiOffset
97+ { apiOffset :: ApiOffset,
91− , provenance :: [Provenance]
98+ provenance :: [Provenance]
92− } deriving (Show,Eq)
100+ deriving (Show, Eq)
94data Config
102data Config
96− { seed :: [Dynamic]
104+ { seed :: [Dynamic],
97− , maxRuntime :: Integer -- seconds to test for
105+ maxRuntime :: Integer, -- seconds to test for
98− , maxReps :: Integer
106+ maxReps :: Integer,
99− , rngSeed :: Int
107+ rngSeed :: Int,
100− , coverageThreshold :: Double
108+ coverageThreshold :: Double
103−data FuzzState = FuzzState
111+data FuzzState
104− { path :: [FuzzOp]
113+ { path :: [FuzzOp],
105− , stash :: Stash
114+ stash :: Stash,
106− , currentRng :: StdGen
115+ currentRng :: StdGen
108− deriving Show
117+ deriving (Show)
110−data EndpointOption = forall as. (V.RecordToList as, V.RMap as) => EndpointOption
121+ (V.RecordToList as, V.RMap as) =>
111− { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
123+ { eoCall :: V.Curried as (IO (Either ServerError (NonEmpty Dynamic))),
112− , eoArgs :: V.Rec (TypedF StashValue) as
124+ eoArgs :: V.Rec (TypedF StashValue) as
115data StopReason
127data StopReason
116 = TimedOut
128 = TimedOut
117 | HitMaxIterations
129 | HitMaxIterations
118− deriving (Show,Eq)
130+ deriving (Show, Eq)
120data Report = Report
132data Report = Report
121 { textual :: String
133 { textual :: String
122 , rsException :: RoboservantException}
134 , rsException :: RoboservantException}
123 deriving (Show)
135 deriving (Show)
125−fuzz :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api))
139+ (FlattenServer api, ToReifiedApi (Endpoints api)) =>
126− => Server api
140+ Server api ->
127− -> Config
141+ Config ->
128− -> IO ()
142+ IO () ->
129− -> IO (Maybe Report)
143+ IO (Maybe Report)
130−fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
144+fuzz server Config {..} checker = handle (pure . Just . formatException) $ do
131 let path = []
145 let path = []
132 stash = addToStash seed mempty
146 stash = addToStash seed mempty
133 currentRng = mkStdGen rngSeed
147 currentRng = mkStdGen rngSeed
135− print ("Initial seed", seed, stash)
136 deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
148 deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
137− (stopreason, fs ) <- runStateT
149+ (stopreason, _fs) <-
138− (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get)) FuzzState{..}
151+ (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
139 print stopreason
153 print stopreason
140 pure Nothing
154 pure Nothing
@@ -174,9 +188,8 @@ fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
174 if now > deadline
188 if now > deadline
175 then pure TimedOut
189 then pure TimedOut
177− action
191+ _ <- action
178− untilDone (n-1, deadline) action
192+ untilDone (n -1, deadline) action
180 reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
193 reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
181 routeCount = length reifiedApi
194 routeCount = length reifiedApi
@@ -185,51 +198,58 @@ fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
185 elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
198 elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
186 elementOrFail l = do
199 elementOrFail l = do
187 st <- get
200 st <- get
188− let (index,newGen) = randomR (0, length l - 1) (currentRng st)
201+ let (index, newGen) = randomR (0, length l - 1) (currentRng st)
189− modify' $ \st' -> st' { currentRng = newGen }
202+ modify' $ \st' -> st' {currentRng = newGen}
190 pure (l !! index)
203 pure (l !! index)
192− withOp :: (MonadState FuzzState m, MonadIO m)
205+ (MonadState FuzzState m, MonadIO m) =>
193− => (forall as. (V.RecordToList as, V.RMap as)
207+ (V.RecordToList as, V.RMap as) =>
194− => FuzzOp
208+ FuzzOp ->
195− -> V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
209+ V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
196− -> V.Rec (TypedF V.Identity) as -> m r
210+ V.Rec (TypedF V.Identity) as ->
199 withOp callback = do
214 withOp callback = do
200 -- choose a call to make, from the endpoints with fillable arguments.
215 -- choose a call to make, from the endpoints with fillable arguments.
201− (offset, EndpointOption{..}) <- elementOrFail . options =<< get
216+ (offset, EndpointOption {..}) <- elementOrFail . options =<< get
202− r <- V.rtraverse
218+ V.rtraverse
203− (\(tr :*: StashValue svs) -> elementOrFail $
219+ ( \(tr :*: StashValue svs) ->
204− zipWith (\i xy -> V.Const i :*: tr :*: xy)
222+ (\i xy -> V.Const i :*: tr :*: xy)
206− (NEL.toList svs)
224+ (NEL.toList svs)
209− let pathSegment = FuzzOp offset $
227+ let pathSegment =
210− recordToList'
229+ recordToList'
211− (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
230+ (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
234+ (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
214− (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
216− modify' (\f -> f { path = path f <> [pathSegment] })
236+ modify' (\f -> f {path = path f <> [pathSegment]})
217 callback pathSegment eoCall argValues
237 callback pathSegment eoCall argValues
219 options :: FuzzState -> [(ApiOffset, EndpointOption)]
239 options :: FuzzState -> [(ApiOffset, EndpointOption)]
220− options FuzzState{..} =
240+ options FuzzState {..} =
222− ( \(offset, ReifiedEndpoint{..}) -> do
242+ ( \(offset, ReifiedEndpoint {..}) -> do
223− args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
243+ args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
224− pure (offset, EndpointOption reEndpointFunc args)
244+ pure (offset, EndpointOption reEndpointFunc args)
226 reifiedApi
246 reifiedApi
228− execute :: (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as)
248+ (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
229− => FuzzOp
249+ FuzzOp ->
230− -> V.Curried as (IO (Either ServerError (NonEmpty Dynamic)))
250+ V.Curried as (IO (Either ServerError (NonEmpty Dynamic))) ->
231− -> V.Rec (TypedF V.Identity) as
251+ V.Rec (TypedF V.Identity) as ->
233 execute fuzzop func args = do
253 execute fuzzop func args = do
234 (liftIO . print . (fuzzop,) . stash ) =<< get
254 (liftIO . print . (fuzzop,) . stash ) =<< get
235 st <- get
255 st <- get
@@ -247,7 +267,7 @@ fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
247 case errHTTPCode serverError of
267 case errHTTPCode serverError of
248 500 -> throw serverError
268 500 -> throw serverError
250− liftIO $ print ("ignoring non-500 error" , serverError)
270+ liftIO $ print ("ignoring non-500 error", serverError)
251 Right (dyn :: NEL.NonEmpty Dynamic) -> do
271 Right (dyn :: NEL.NonEmpty Dynamic) -> do
252 -- liftIO $ print ("storing", fmap dynTypeRep dyn)
272 -- liftIO $ print ("storing", fmap dynTypeRep dyn)
253 modify' (\fs@FuzzState{..} ->
273 modify' (\fs@FuzzState{..} ->
@@ -255,9 +275,9 @@ fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
256 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
276 argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
257 argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
277 argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
259− go :: (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m)
279+ (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
261 go = withOp $ \op func args -> do
281 go = withOp $ \op func args -> do
262 catches (execute op func args)
282 catches (execute op func args)
263 [ Handler (\(e :: SomeAsyncException) -> throw e)
283 [ Handler (\(e :: SomeAsyncException) -> throw e)
@@ -268,29 +288,33 @@ fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
268 catch (liftIO checker)
288 catch (liftIO checker)
269 (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
289 (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
271−addToStash :: [Dynamic]
291+addToStash ::
272− -> Stash
292+ [Dynamic] ->
274addToStash result stash =
295addToStash result stash =
275− foldr (\(Dynamic tr x) (Stash dict) -> Stash $
297+ ( \(Dynamic tr x) (Stash dict) ->
276− DM.insertWith renumber tr (StashValue (([Provenance (R.SomeTypeRep tr) 0],x):|[])) dict
299+ DM.insertWith renumber tr (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| [])) dict
281− renumber :: StashValue a
305+ StashValue a ->
282− -> StashValue a
306+ StashValue a ->
283− -> StashValue a
307+ StashValue a
284 renumber (StashValue singleDyn) (StashValue l) = StashValue $ case NEL.toList singleDyn of
308 renumber (StashValue singleDyn) (StashValue l) = StashValue $ case NEL.toList singleDyn of
285− [([Provenance tr _], dyn)] -> l
309+ [([Provenance tr _], dyn)] ->
286− <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
311+ <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
287 _ -> error "should be impossible"
312 _ -> error "should be impossible"
290-- why isn't this in vinyl?
314-- why isn't this in vinyl?
291−recordToList'
315+recordToList' ::
292− :: (V.RecordToList as, V.RMap as)
316+ (V.RecordToList as, V.RMap as) =>
293− => (forall x. f x -> a)
317+ (forall x. f x -> a) ->
294− -> V.Rec f as
318+ V.Rec f as ->
296recordToList' f = V.recordToList . V.rmap (V.Const . f)
320recordToList' f = V.recordToList . V.rmap (V.Const . f)