@@ -1,81 +1,60 @@
1{-# LANGUAGE AllowAmbiguousTypes #-}
1{-# LANGUAGE AllowAmbiguousTypes #-}
2+{-# LANGUAGE DataKinds #-}
3+{-# LANGUAGE DerivingStrategies #-}
2{-# LANGUAGE FlexibleContexts #-}
4{-# LANGUAGE FlexibleContexts #-}
5+{-# LANGUAGE FlexibleInstances #-}
3{-# LANGUAGE GADTs #-}
6{-# LANGUAGE GADTs #-}
7+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
4{-# LANGUAGE LambdaCase #-}
8{-# LANGUAGE LambdaCase #-}
5{-# LANGUAGE PolyKinds #-}
9{-# LANGUAGE PolyKinds #-}
6−{-# LANGUAGE RankNTypes #-}
7{-# LANGUAGE RecordWildCards #-}
10{-# LANGUAGE RecordWildCards #-}
8{-# LANGUAGE ScopedTypeVariables #-}
11{-# LANGUAGE ScopedTypeVariables #-}
12+{-# LANGUAGE StandaloneDeriving #-}
9{-# LANGUAGE TupleSections #-}
13{-# LANGUAGE TupleSections #-}
10−
14+{-# LANGUAGE TypeApplications #-}
11{-# LANGUAGE TypeFamilies #-}
15{-# LANGUAGE TypeFamilies #-}
16+{-# LANGUAGE TypeOperators #-}
12{-# LANGUAGE UndecidableInstances #-}
17{-# LANGUAGE UndecidableInstances #-}
14module Roboservant.Direct
19module Roboservant.Direct
16− Config (..),
20+ ( fuzz, Config(..)
17− -- TODO come up with something smarter than exporting all this, we should
22+ -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
23+ -- have some nice error-display functions
19− RoboservantException (..),
24+ , RoboservantException(..), FuzzState(..), FuzzOp(..), FailureType(..), Report(..)
27−import Control.Exception.Lifted
28+import Control.Exception.Lifted(throw,handle,Handler(..), Exception,SomeException,SomeAsyncException, catch, catches)
28− ( Exception,
29+import Control.Monad(void,replicateM)
37−import Control.Monad.State.Strict
30+import Control.Monad.State.Strict(MonadState,MonadIO,get,modify',liftIO,runStateT)
43−import Control.Monad.Trans.Control (MonadBaseControl)
31+import Control.Monad.Trans.Control(MonadBaseControl)
44−import qualified Data.Dependent.Map as DM
45−import Data.Dynamic (Dynamic (..))
32+import Data.Dynamic (Dynamic, dynApply, dynTypeRep, fromDynamic)
46−import qualified Data.IntSet as IntSet
33+import Data.Map.Strict(Map)
47−import Data.List.NonEmpty (NonEmpty (..))
48−import qualified Data.List.NonEmpty as NEL
49−import Data.Maybe (mapMaybe)
50import qualified Data.Set as Set
34import qualified Data.Set as Set
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
35+import Data.Maybe (mapMaybe)
36+import Data.Typeable (TypeRep)
37+import Servant (Endpoints, Proxy (Proxy), Server, ServerError(..))
38+import System.Random(StdGen,randomR,mkStdGen)
39+import qualified Data.List.NonEmpty as NEL
40+import qualified Data.Map.Strict as Map
41+import Data.Time.Clock
52−import qualified Data.Vinyl as V
42+import Data.List(sortOn)
53−import qualified Data.Vinyl.Curry as V
43+
54−import qualified Data.Vinyl.Functor as V
44+import Roboservant.Types.Breakdown
55−import GHC.Generics ((:*:) (..))
56import Roboservant.Types
45import Roboservant.Types
57 ( ApiOffset (..),
46 ( ApiOffset (..),
58− Argument (..),
47+ FlattenServer (..),
59− InteractionError(..),
61− ReifiedApi,
48+-- ReifiedApi,
62− ReifiedEndpoint (..),
49+ ToReifiedApi (..),
67−import Roboservant.Types.Config
69−import System.Random (Random (randomR), StdGen, mkStdGen)
70−import qualified Type.Reflection as R
72data RoboservantException
52data RoboservantException
73 = RoboservantException
53 = RoboservantException
74− { failureReason :: FailureType,
54+ { failureReason :: FailureType
75− serverException :: Maybe SomeException,
55+ , serverException :: Maybe SomeException
76− fuzzState :: FuzzState
56+ , fuzzState :: FuzzState
78− deriving (Show)
57+ } deriving (Show)
80instance Exception RoboservantException
59instance Exception RoboservantException
@@ -84,228 +63,192 @@ data FailureType
84 | CheckerFailed
63 | CheckerFailed
85 | NoPossibleMoves
64 | NoPossibleMoves
86 | InsufficientCoverage Double
65 | InsufficientCoverage Double
87− deriving (Show, Eq)
66+ deriving (Show,Eq)
89−data FuzzOp
69+data FuzzOp = FuzzOp
91− { apiOffset :: ApiOffset,
70+ { apiOffset :: ApiOffset
92− provenance :: [Provenance]
71+ , provenance :: [Provenance]
94− deriving (Show, Eq)
72+ } deriving (Show,Eq)
96−data FuzzState
74+data Config
77+ , maxRuntime :: Integer -- seconds to test for
80+ , coverageThreshold :: Double
97− = FuzzState
83+data FuzzState = FuzzState
98− { path :: [FuzzOp],
84+ { path :: [FuzzOp]
99− stash :: Stash,
85+ , stash :: Stash
100− currentRng :: StdGen
86+ , currentRng :: StdGen
102 deriving (Show)
88 deriving (Show)
106− (V.RecordToList as, V.RMap as) =>
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
109− eoArgs :: V.Rec (TypedF StashValue) as
112data StopReason
90data StopReason
113 = TimedOut
91 = TimedOut
114 | HitMaxIterations
92 | HitMaxIterations
115− deriving (Show, Eq)
93+ deriving (Show,Eq)
117−data Report
95+data Report = Report
119− { textual :: String,
96+ { textual :: String
120− rsException :: RoboservantException
97+ , rsException :: RoboservantException}
122− deriving (Show)
98+ deriving (Show)
124−
100+fuzz :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api))
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
102+ -> Config
127−-- fuzzClient = undefined
103+ -> IO ()
134− IO (Maybe Report)
104+ -> IO (Maybe Report)
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
105+fuzz server Config{..} checker = handle (pure . Just . formatException) $ do
136 let path = []
106 let path = []
137 stash = addToStash seed mempty
107 stash = addToStash seed mempty
138 currentRng = mkStdGen rngSeed
108 currentRng = mkStdGen rngSeed
139− deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
111+ deadline :: UTCTime <- addUTCTime (fromInteger $ maxRuntime * 1000000) <$> getCurrentTime
140− (stopreason, _fs) <-
141− runStateT
112+ (stopreason, fs ) <- runStateT
142− (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
113+ (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get)) FuzzState{..}
144− logInfo $ show stopreason
145 pure Nothing
114 pure Nothing
115+ -- mapM_ (print . (\(offset, (args, dyn) ) -> (offset, map fst args, dyn))) reifiedApi
147 -- something less terrible later
118 -- something less terrible later
148 formatException :: RoboservantException -> Report
119 formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
120+ formatException r@(RoboservantException failureType exception _state) = Report
151− (unlines [show failureType, show exception])
121+ (unlines [ show failureType , show exception ])
153− displayDiagnostics FuzzState {..} = liftIO $
124+ displayDiagnostics FuzzState{..} = liftIO $ do
155− ["api endpoints covered"]
125+ putStrLn "api endpoints covered"
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
126+ mapM_ print (Set.toList $ Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
128+ putStrLn "types in stash"
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
129+ mapM_ print (map (\x -> (fst x, NEL.length (snd x))) . sortOn (show . fst) . Map.toList $ stash)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
130+
160− -- $ \_k v ->
131+ -- evaluateCoverage :: FuzzState -> m ()
161− -- (show . NEL.length $ getStashValue v))
163− evaluateCoverage f@FuzzState {..}
132+ evaluateCoverage f@FuzzState{..}
164 | coverage > coverageThreshold = pure ()
133 | coverage > coverageThreshold = pure ()
165 | otherwise = do
134 | otherwise = do
166− displayDiagnostics f
135+ displayDiagnostics f
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
136+ throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
137+ where hitRoutes = (fromIntegral . Set.size . Set.fromList $ map apiOffset path)
170− totalRoutes = fromIntegral routeCount
138+ totalRoutes = (fromIntegral routeCount)
171− coverage = hitRoutes / totalRoutes
139+ coverage = hitRoutes / totalRoutes
172− untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
142+ untilDone :: MonadIO m => (Integer,UTCTime) -> m a -> m StopReason
173− untilDone (0, _) _ = pure HitMaxIterations
143+ untilDone (0,_) _ = pure HitMaxIterations
174 untilDone (n, deadline) action = do
144 untilDone (n, deadline) action = do
175 now <- liftIO getCurrentTime
145 now <- liftIO getCurrentTime
176 if now > deadline
146 if now > deadline
177 then pure TimedOut
147 then pure TimedOut
179− _ <- action
149+ action
180− untilDone (n -1, deadline) action
150+ untilDone (n-1, deadline) action
152+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
182 routeCount = length reifiedApi
153 routeCount = length reifiedApi
183− elementOrFail ::
154+
184− (MonadState FuzzState m, MonadIO m) =>
155+ elementOrFail :: (MonadState FuzzState m, MonadIO m)
186− m a
156+ => [a] -> m a
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
157+ elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
188 elementOrFail l = do
158 elementOrFail l = do
189 st <- get
159 st <- get
190− let (index, newGen) = randomR (0, length l - 1) (currentRng st)
160+ let (index,newGen) = randomR (0, length l - 1) (currentRng st)
191− modify' $ \st' -> st' {currentRng = newGen}
161+ modify' $ \st' -> st' { currentRng = newGen }
192 pure (l !! index)
162 pure (l !! index)
194− (MonadState FuzzState m, MonadIO m) =>
164+ genOp :: (MonadState FuzzState m, MonadIO m)
196− (V.RecordToList as, V.RMap as) =>
197− FuzzOp ->
165+ => m (FuzzOp, Dynamic, [Dynamic])
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
166+ genOp = do -- fs@FuzzState{..} = do
199− V.Rec (TypedF V.Identity) as ->
203− withOp callback = do
204 -- choose a call to make, from the endpoints with fillable arguments.
167 -- choose a call to make, from the endpoints with fillable arguments.
205− (offset, EndpointOption {..}) <- elementOrFail . options =<< get
168+ (offset, dynCall, args) <- elementOrFail . options =<< get
206− r <-
169+ r <- mapM (elementOrFail . zip [0..] . NEL.toList) args
207− V.rtraverse
170+ let pathSegment = FuzzOp offset (map (\(index,(_,dyn) ) -> Provenance (dynTypeRep dyn) index) r)
208− ( \(tr :*: StashValue svs _) ->
211− (\i xy -> V.Const i :*: tr :*: xy)
219− (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
223− (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
225− modify' (\f -> f {path = path f <> [pathSegment]})
171+ modify' (\f -> f { path = path f <> [pathSegment] })
226− callback pathSegment eoCall argValues
172+ pure (pathSegment, dynCall, fmap (snd . snd) r)
228− options :: FuzzState -> [(ApiOffset, EndpointOption)]
175+ options :: FuzzState -> [(ApiOffset, Dynamic, [NEL.NonEmpty ([Provenance], Dynamic)])]
229− options FuzzState {..} =
176+ options FuzzState{..} =
231− ( \(offset, ReifiedEndpoint {..}) -> do
178+ ( \(offset, (argreps, dynCall)) -> (offset,dynCall,) <$> do
232− args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
179+ mapM (\(_tr,bf) -> bf stash ) argreps
233− pure (offset, EndpointOption reEndpointFunc args)
235 reifiedApi
181 reifiedApi
237− (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
240− V.Rec (TypedF V.Identity) as ->
242− execute fuzzop func args = do
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
244− liftIO (V.runcurry' func argVals) >>= \case
245− Left (e::InteractionError) ->
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
251− ( \fs@FuzzState {..} ->
252− fs {stash = addToStash (NEL.toList dyn) stash}
255− argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
258− (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
260− go = withOp $ \op func args -> do
262− (execute op func args)
263− [ Handler (\(e :: SomeAsyncException) -> throw e),
265− ( \(e :: SomeException) ->
266− throw . RoboservantException ServerCrashed (Just e) =<< get
270− (liftIO healthCheck)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
273−addToStash ::
183+ execute :: (MonadState FuzzState m, MonadIO m)
274− [(Dynamic, Int)] ->
184+ => (FuzzOp,Dynamic,[Dynamic]) -> m ()
275− Stash ->
185+ execute (fuzzop, dyncall, args) = do
276− Stash
186+ -- liftIO $ print fuzzop
187+ -- now, magic happens: we apply some dynamic arguments to a dynamic
188+ -- function and hopefully something useful pops out the end.
189+ let func = foldr (\arg curr -> flip dynApply arg =<< curr) (Just dyncall) (reverse args)
191+ let showable = unlines $ ("args":map (show . dynTypeRep) args)
195+ ,show (dynTypeRep dyncall)
198+ -- liftIO $ putStrLn showable
201+ Nothing -> error ("all screwed up 1: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
204+ case fromDynamic f' of
205+ Nothing -> error ("all screwed up 2: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
206+ Just (f) -> liftIO f >>= \case
207+ -- parameterise this
208+ Left (serverError :: ServerError) ->
209+ case errHTTPCode serverError of
210+ 500 -> throw serverError
212+ liftIO $ print ("ignoring non-500 error" , serverError)
214+ Right (dyn :: NEL.NonEmpty Dynamic) -> do
215+ -- liftIO $ print ("storing", fmap dynTypeRep dyn)
216+ modify' (\fs@FuzzState{..} ->
217+ fs { stash = addToStash (NEL.toList dyn) stash } )
220+-- go :: (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m)
224+ catches (execute op)
225+ [ Handler (\(e :: SomeAsyncException) -> throw e)
226+ , Handler (\(e :: SomeException) -> do
227+ displayDiagnostics =<< get
228+ throw . RoboservantException ServerCrashed (Just e) =<< get)
230+ catch (liftIO checker)
231+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
236+ -- (Range.linear 1 100)
238+ -- (fmap preload seed <> [callEndpoint reifiedApi])
239+ -- executeSequential emptyState actions
240+addToStash :: [Dynamic]
241+ -> Map TypeRep (NEL.NonEmpty ([Provenance], Dynamic))
242+ -> Map TypeRep (NEL.NonEmpty ([Provenance], Dynamic))
277addToStash result stash =
243addToStash result stash =
278− foldr
244+ foldr (\dyn dict -> let tr = dynTypeRep dyn in
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
245+ Map.insertWith renumber tr (pure ([Provenance tr 0],dyn)) dict) stash result
280− Stash $
246+-- Map.insertWith (flip (<>)) (dynTypeRep result) (_pure result) stash })
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
290− renumber ::
248+ renumber :: NEL.NonEmpty ([Provenance],Dynamic)
291− StashValue a ->
249+ -> NEL.NonEmpty ([Provenance],Dynamic)
292− StashValue a ->
250+ -> NEL.NonEmpty ([Provenance],Dynamic)
294− renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
295− | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
298− ( case NEL.toList singleDyn of
251+ renumber singleDyn l = case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
252+ [([Provenance tr _], dyn)] -> l
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
253+ <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
301− _ -> error "should be impossible"
254+ _ -> error "should be impossible"
303− (IntSet.union singleHash intSet)
305−-- why isn't this in vinyl?
307− (V.RecordToList as, V.RMap as) =>
308− (forall x. f x -> a) ->
311−recordToList' f = V.recordToList . V.rmap (V.Const . f)