@@ -1,81 +1,56 @@
11{-# LANGUAGE AllowAmbiguousTypes #-}
2+{-# LANGUAGE DataKinds #-}
3+{-# LANGUAGE DerivingStrategies #-}
24{-# LANGUAGE FlexibleContexts #-}
5+{-# LANGUAGE FlexibleInstances #-}
36{-# LANGUAGE GADTs #-}
47{-# LANGUAGE LambdaCase #-}
58{-# LANGUAGE PolyKinds #-}
6−{-# LANGUAGE RankNTypes #-}
79{-# LANGUAGE RecordWildCards #-}
810{-# LANGUAGE ScopedTypeVariables #-}
911{-# LANGUAGE TupleSections #-}
10−
12+{-# LANGUAGE TypeApplications #-}
1113{-# LANGUAGE TypeFamilies #-}
14+
1215{-# LANGUAGE UndecidableInstances #-}
1316
1417module Roboservant.Direct
15− ( fuzz',
16− Config (..),
17− -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
19− RoboservantException (..),
20− FuzzState (..),
21− FuzzOp (..),
22− FailureType (..),
23− Report (..),
18+ ( fuzz, Config(..)
19+
20+ -- TODO come up with something smarter than exporting all this, we should
21+ -- have some nice error-display functions
22+ , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
2423 )
2524where
2625
27−import Control.Exception.Lifted
28− ( Exception,
29− Handler (Handler),
30− SomeAsyncException,
31− SomeException,
32− catch,
33− catches,
34− handle,
35− throw,
36− )
37−import Control.Monad.State.Strict
38− ( MonadIO (..),
39− MonadState (get),
40− StateT (runStateT),
41− modify',
42− )
43−import Control.Monad.Trans.Control (MonadBaseControl)
44−import qualified Data.Dependent.Map as DM
45−import Data.Dynamic (Dynamic (..))
46−import qualified Data.IntSet as IntSet
47−import Data.List.NonEmpty (NonEmpty (..))
48−import qualified Data.List.NonEmpty as NEL
49−import Data.Maybe (mapMaybe)
26+import Control.Exception.Lifted(throw,handle,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
27+import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,runStateT)
28+import Data.Dynamic (Dynamic, dynApply, dynTypeRep, fromDynamic)
29+import Data.Map.Strict(Map)
5030import qualified Data.Set as Set
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
52−import qualified Data.Vinyl as V
53−import qualified Data.Vinyl.Curry as V
54−import qualified Data.Vinyl.Functor as V
55−import GHC.Generics ((:*:) (..))
31+import Data.Maybe (mapMaybe)
32+import Data.Typeable (TypeRep)
33+import Servant (Endpoints, Proxy (Proxy), Server, ServerError(..))
34+import System.Random(StdGen,randomR,mkStdGen)
35+import qualified Data.List.NonEmpty as NEL
36+import qualified Data.Map.Strict as Map
37+import Data.Time.Clock
38+import Data.List(sortOn)
39+import Control.Arrow(second)
40+
41+import Roboservant.Types.Breakdown
5642import Roboservant.Types
5743 ( ApiOffset (..),
58− Argument (..),
59− InteractionError(..),
60− Provenance (..),
61− ReifiedApi,
62− ReifiedEndpoint (..),
63− Stash (..),
64− StashValue (..),
65− TypedF,
66− )
67−import Roboservant.Types.Config
68−
69−import System.Random (Random (randomR), StdGen, mkStdGen)
70−import qualified Type.Reflection as R
71−
44+ FlattenServer (..),
45+-- ReifiedApi,
46+ ToReifiedApi (..),
47+ ))
7248data RoboservantException
7349 = RoboservantException
74− { failureReason :: FailureType,
75− serverException :: Maybe SomeException,
76− fuzzState :: FuzzState
77− }
78− deriving (Show)
50+ { failureReason :: FailureType
51+ , serverException :: Maybe SomeException
52+ , fuzzState :: FuzzState
53+ } deriving (Show)
7954
8055instance Exception RoboservantException
8156
@@ -84,228 +59,194 @@ data FailureType
8459 | CheckerFailed
8560 | NoPossibleMoves
8661 | InsufficientCoverage Double
87− deriving (Show, Eq)
88−
89−data FuzzOp
90− = FuzzOp
91− { apiOffset :: ApiOffset,
92− provenance :: [Provenance]
93− }
94− deriving (Show, Eq)
95−
96−data FuzzState
97− = FuzzState
98− { path :: [FuzzOp],
99− stash :: Stash,
100− currentRng :: StdGen
101− }
62+ deriving (Show,Eq)
63+
64+
65+data FuzzOp = FuzzOp
66+ { apiOffset :: ApiOffset
67+ , provenance :: [Provenance]
68+ } deriving (Show,Eq)
69+
70+data Config
71+ = Config
72+ { seed :: [Dynamic]
73+ , maxRuntime :: Integer -- seconds to test for
74+ , maxReps :: Integer
75+ , rngSeed :: Int
76+ , coverageThreshold :: Double
77+ }
78+
79+data FuzzState = FuzzState
80+ { path :: [FuzzOp]
81+ , stash :: Stash
82+ , currentRng :: StdGen
83+ }
10284 deriving (Show)
10385
104−data EndpointOption
105− = forall as.
106− (V.RecordToList as, V.RMap as) =>
107− EndpointOption
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
109− eoArgs :: V.Rec (TypedF StashValue) as
110− }
111−
11286data StopReason
11387 = TimedOut
11488 | HitMaxIterations
115− deriving (Show, Eq)
89+ deriving (Show,Eq)
11690
117−data Report
118− = Report
119− { textual :: String,
120− rsException :: RoboservantException
121− }
91+data Report = Report
92+ { textual :: String
93+ , rsException :: RoboservantException}
12294 deriving (Show)
12395
124−
125−
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127−-- fuzzClient = undefined
128−
129−
130−
131−fuzz' ::
132− ReifiedApi ->
133− Config ->
134− IO (Maybe Report)
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
96+fuzz :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api))
97+ => Server api
98+ -> Config
99+ -> IO ()
100+ -> IO (Maybe Report)
101+fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
136102 let path = []
137103 stash = addToStash seed mempty
138104 currentRng = mkStdGen rngSeed
139− deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
140− (stopreason, _fs) <-
141− runStateT
142− (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
143− FuzzState {..}
144− logInfo $ show stopreason
105+
106+
107+ deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
108+ (stopreason, fs ) <- runStateT
109+ (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get)) FuzzState{..}
145110 pure Nothing
111+ -- mapM_ (print . (\(offset, (args, dyn) ) -> (offset, map fst args, dyn))) reifiedApi
112+
146113 where
147114 -- something less terrible later
148115 formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
150− Report
151− (unlines [show failureType, show exception])
152− r
153− displayDiagnostics FuzzState {..} = liftIO $
154− logInfo $ unlines $
155− ["api endpoints covered"]
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
160− -- $ \_k v ->
161− -- (show . NEL.length $ getStashValue v))
162−
163− evaluateCoverage f@FuzzState {..}
116+ formatException r@(RoboservantException failureType exception _state) = Report
117+ (unlines [ show failureType , show exception ])
118+ r
119+
120+ displayDiagnostics FuzzState{..} = liftIO $ do
121+ putStrLn "api endpoints covered"
122+ mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
123+ putStrLn ""
124+ putStrLn "types in stash"
125+ mapM_
126+ (print . second NEL.length)
127+ (sortOn (show . fst) . Map.toList stash)
128+
129+ -- evaluateCoverage :: FuzzState -> m ()
130+ evaluateCoverage f@FuzzState{..}
164131 | coverage > coverageThreshold = pure ()
165132 | otherwise = do
166− displayDiagnostics f
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
168− where
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
170− totalRoutes = fromIntegral routeCount
171− coverage = hitRoutes / totalRoutes
172− untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
173− untilDone (0, _) _ = pure HitMaxIterations
133+ displayDiagnostics f
134+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
135+ where hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
136+ totalRoutes = fromIntegral routeCount
137+ coverage = hitRoutes / totalRoutes
138+
139+
140+ untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
141+ untilDone (0,_) _ = pure HitMaxIterations
174142 untilDone (n, deadline) action = do
175143 now <- liftIO getCurrentTime
176144 if now > deadline
177145 then pure TimedOut
178146 else do
179− _ <- action
180− untilDone (n -1, deadline) action
147+ action
148+ untilDone (n-1, deadline) action
181149
150+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
182151 routeCount = length reifiedApi
183− elementOrFail ::
184− (MonadState FuzzState m, MonadIO m) =>
185− [a] ->
186− m a
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
152+
153+ elementOrFail :: (MonadState FuzzState m, MonadIO m)
154+ => [a] -> m a
155+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
188156 elementOrFail l = do
189157 st <- get
190− let (index, newGen) = randomR (0, length l - 1) (currentRng st)
191− modify' $ \st' -> st' {currentRng = newGen}
158+ let (index,newGen) = randomR (0, length l - 1) (currentRng st)
159+ modify' $ \st' -> st' { currentRng = newGen }
192160 pure (l !! index)
193− withOp ::
194− (MonadState FuzzState m, MonadIO m) =>
195− ( forall as.
196− (V.RecordToList as, V.RMap as) =>
197− FuzzOp ->
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
199− V.Rec (TypedF V.Identity) as ->
200− m r
201− ) ->
202− m r
203− withOp callback = do
161+
162+ genOp :: (MonadState FuzzState m, MonadIO m)
163+ => m (FuzzOp, Dynamic, [Dynamic])
164+ genOp = do -- fs@FuzzState{..} = do
204165 -- choose a call to make, from the endpoints with fillable arguments.
205− (offset, EndpointOption {..}) <- elementOrFail . options =<< get
206− r <-
207− V.rtraverse
208− ( \(tr :*: StashValue svs _) ->
209− elementOrFail $
210− zipWith
211− (\i xy -> V.Const i :*: tr :*: xy)
212− [0 ..]
213− (NEL.toList svs)
214− )
215− eoArgs
216− let pathSegment =
217− FuzzOp offset $
218− recordToList'
219− (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
220− r
221− argValues =
222− V.rmap
223− (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
224− r
225− modify' (\f -> f {path = path f <> [pathSegment]})
226− callback pathSegment eoCall argValues
166+ (offset, dynCall, args) <- elementOrFail . options =<< get
167+ r <- mapM (elementOrFail . zip [0..] . NEL.toList) args
168+ let pathSegment = FuzzOp offset (map (\(index,(_,dyn) ) -> Provenance (dynTypeRep dyn) index) r)
169+ modify' (\f -> f { path = path f <> [pathSegment] })
170+ pure (pathSegment, dynCall, fmap (snd . snd) r)
171+
227172 where
228− options :: FuzzState -> [(ApiOffset, EndpointOption)]
229− options FuzzState {..} =
173+ options :: FuzzState -> [(ApiOffset, Dynamic, [NEL.NonEmpty ([Provenance], Dynamic)])]
174+ options FuzzState{..} =
230175 mapMaybe
231− ( \(offset, ReifiedEndpoint {..}) -> do
232− args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
233− pure (offset, EndpointOption reEndpointFunc args)
176+ ( \(offset, (argreps, dynCall)) -> (offset,dynCall,) <$>
177+ mapM (\(_tr,bf) -> bf stash ) argreps
234178 )
235179 reifiedApi
236− execute ::
237− (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
238− FuzzOp ->
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
240− V.Rec (TypedF V.Identity) as ->
241− m ()
242− execute fuzzop func args = do
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
244− liftIO (V.runcurry' func argVals) >>= \case
245− Left (e::InteractionError) ->
246− if fatalError e
247− then throw e
248− else pure ()
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
250− modify'
251− ( \fs@FuzzState {..} ->
252− fs {stash = addToStash (NEL.toList dyn) stash}
253− )
254− where
255− argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
257− go ::
258− (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
259− m ()
260− go = withOp $ \op func args -> do
261− catches
262− (execute op func args)
263− [ Handler (\(e :: SomeAsyncException) -> throw e),
264− Handler
265− ( \(e :: SomeException) ->
266− throw . RoboservantException ServerCrashed (Just e) =<< get
267− )
268− ]
269− catch
270− (liftIO healthCheck)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
272180
273−addToStash ::
274− [(Dynamic, Int)] ->
275− Stash ->
276− Stash
181+ execute :: (MonadState FuzzState m, MonadIO m)
182+ => (FuzzOp,Dynamic,[Dynamic]) -> m ()
183+ execute (fuzzop, dyncall, args) = do
184+ -- liftIO $ print fuzzop
185+ -- now, magic happens: we apply some dynamic arguments to a dynamic
186+ -- function and hopefully something useful pops out the end.
187+ let func = foldr (\arg curr -> flip dynApply arg =<< curr) (Just dyncall) (reverse args)
188+ st <- get
189+ let showable = unlines $ ("args":map (show . dynTypeRep) args)
190+ <> ["fuzzop"
191+ , show fuzzop
192+ ,"dyncall"
193+ ,show (dynTypeRep dyncall)
194+ ,"state"
195+ ,show st]
196+ -- liftIO $ putStrLn showable
197+
198+ case func of
199+ Nothing -> error ("all screwed up 1: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
200+ Just f' -> do
201+ -- liftIO $ print
202+ case fromDynamic f' of
203+ Nothing -> error ("all screwed up 2: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
204+ Just f -> liftIO f >>= \case
205+ -- parameterise this
206+ Left (serverError :: ServerError) ->
207+ case errHTTPCode serverError of
208+ 500 -> throw serverError
209+ _ ->
210+ liftIO $ print ("ignoring non-500 error" , serverError)
211+
212+ Right (dyn :: NEL.NonEmpty Dynamic) -> do
213+ -- liftIO $ print ("storing", fmap dynTypeRep dyn)
214+ modify' (\fs@FuzzState{..} ->
215+ fs { stash = addToStash (NEL.toList dyn) stash } )
216+ pure ()
217+
218+-- go :: (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m)
219+-- => m ()
220+ go = do
221+ op <- genOp
222+ catches (execute op)
223+ [ Handler (\(e :: SomeAsyncException) -> throw e)
224+ , Handler (\(e :: SomeException) -> do
225+ displayDiagnostics =<< get
226+ throw . RoboservantException ServerCrashed (Just e) =<< get)
227+ ]
228+ catch (liftIO checker)
229+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
230+
231+ -- actions <-
232+ -- forAll $ do
233+ -- Gen.sequential
234+ -- (Range.linear 1 100)
235+ -- emptyState
236+ -- (fmap preload seed <> [callEndpoint reifiedApi])
237+ -- executeSequential emptyState actions
238+addToStash :: [Dynamic]
239+ -> Map TypeRep (NEL.NonEmpty ([Provenance], Dynamic))
240+ -> Map TypeRep (NEL.NonEmpty ([Provenance], Dynamic))
277241addToStash result stash =
278− foldr
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
280− Stash $
281− DM.insertWith
282− renumber
283− tr
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
285− dict
286− )
287− stash
288− result
242+ foldr (\dyn dict -> let tr = dynTypeRep dyn in
243+ Map.insertWith renumber tr (pure ([Provenance tr 0],dyn)) dict) stash result
244+-- Map.insertWith (flip (<>)) (dynTypeRep result) (_pure result) stash })
289245 where
290− renumber ::
291− StashValue a ->
292− StashValue a ->
293− StashValue a
294− renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
295− | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
296− | otherwise =
297− StashValue
298− ( case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
301− _ -> error "should be impossible"
302− )
303− (IntSet.union singleHash intSet)
304−
305−-- why isn't this in vinyl?
306−recordToList' ::
307− (V.RecordToList as, V.RMap as) =>
308− (forall x. f x -> a) ->
309− V.Rec f as ->
310− [a]
311−recordToList' f = V.recordToList . V.rmap (V.Const . f)
246+ renumber :: NEL.NonEmpty ([Provenance],Dynamic)
247+ -> NEL.NonEmpty ([Provenance],Dynamic)
248+ -> NEL.NonEmpty ([Provenance],Dynamic)
249+ renumber singleDyn l = case NEL.toList singleDyn of
250+ [([Provenance tr _], dyn)] -> l
251+ <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
252+ _ -> error "should be impossible"