@@ -1,311 +1,229 @@
1{-# LANGUAGE AllowAmbiguousTypes #-}
1{-# LANGUAGE AllowAmbiguousTypes #-}
2+{-# LANGUAGE DataKinds #-}
3+{-# LANGUAGE DerivingStrategies #-}
2{-# LANGUAGE FlexibleContexts #-}
4{-# LANGUAGE FlexibleContexts #-}
5+{-# LANGUAGE FlexibleInstances #-}
3{-# LANGUAGE GADTs #-}
6{-# LANGUAGE GADTs #-}
7+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
4{-# LANGUAGE LambdaCase #-}
8{-# LANGUAGE LambdaCase #-}
5{-# LANGUAGE PolyKinds #-}
9{-# LANGUAGE PolyKinds #-}
6−{-# LANGUAGE RankNTypes #-}
7{-# LANGUAGE RecordWildCards #-}
10{-# LANGUAGE RecordWildCards #-}
8{-# LANGUAGE ScopedTypeVariables #-}
11{-# LANGUAGE ScopedTypeVariables #-}
12+{-# LANGUAGE StandaloneDeriving #-}
9{-# LANGUAGE TupleSections #-}
13{-# LANGUAGE TupleSections #-}
10−
14+{-# LANGUAGE TypeApplications #-}
11{-# LANGUAGE TypeFamilies #-}
15{-# LANGUAGE TypeFamilies #-}
16+{-# LANGUAGE TypeOperators #-}
12{-# LANGUAGE UndecidableInstances #-}
17{-# LANGUAGE UndecidableInstances #-}
14module Roboservant.Direct
19module Roboservant.Direct
17− -- TODO come up with something smarter than exporting all this, we should
18− -- have some nice error-display functions
19− RoboservantException (..),
27−import Control.Exception.Lifted
24+import Control.Monad.Trans.Control
37import Control.Monad.State.Strict
25import Control.Monad.State.Strict
38− ( MonadIO (..),
26+import Control.Exception.Lifted(throw,Exception,SomeException,catch)
39− MonadState (get),
27+import System.Random
40− StateT (runStateT),
28+import System.Timeout.Lifted
41− modify',
29+import Control.Monad (guard)
42− )
30+import Control.Monad.IO.Class (MonadIO, liftIO)
43−import Control.Monad.Trans.Control (MonadBaseControl)
31+import Data.Dynamic (Dynamic, dynApply, dynTypeRep, fromDynamic)
44−import qualified Data.Dependent.Map as DM
45−import Data.Dynamic (Dynamic (..))
46−import qualified Data.IntSet as IntSet
47−import Data.List.NonEmpty (NonEmpty (..))
48import qualified Data.List.NonEmpty as NEL
32import qualified Data.List.NonEmpty as NEL
33+import qualified Data.Map.Strict as Map
34+import Data.Map.Strict(Map)
49import Data.Maybe (mapMaybe)
35import Data.Maybe (mapMaybe)
50−import qualified Data.Set as Set
36+import Data.Typeable (TypeRep)
51−import Data.Time.Clock (UTCTime, addUTCTime, getCurrentTime)
37+import Hedgehog
52−import qualified Data.Vinyl as V
38+ ( Callback (Update),
53−import qualified Data.Vinyl.Curry as V
50+import qualified Hedgehog.Gen as Gen
54−import qualified Data.Vinyl.Functor as V
51+import qualified Hedgehog.Range as Range
55−import GHC.Generics ((:*:) (..))
52+import Roboservant.Hedgehog (elementOrFail)
56import Roboservant.Types
53import Roboservant.Types
57 ( ApiOffset (..),
54 ( ApiOffset (..),
58− Argument (..),
55+ FlattenServer (..),
59− InteractionError(..),
56+ Op (..),
61 ReifiedApi,
57 ReifiedApi,
62− ReifiedEndpoint (..),
58+ State (..),
63− Stash (..),
59+ ToReifiedApi (..),
64− StashValue (..),
60+ emptyState,
67−import Roboservant.Types.Config
62+import Servant (Endpoints, Proxy (Proxy), Server, ServerError)
63+import Type.Reflection (SomeTypeRep)
69−import System.Random (Random (randomR), StdGen, mkStdGen)
65+-- prop_concurrent,
66+import Control.Monad (guard)
67+import Control.Monad.IO.Class (MonadIO, liftIO)
68+import Data.Dynamic (Dynamic, dynApply, dynTypeRep, fromDynamic)
69+import qualified Data.List.NonEmpty as NEL
70+import qualified Data.Map.Strict as Map
71+import Data.Maybe (mapMaybe)
72+import Data.Typeable (TypeRep)
70−import qualified Type.Reflection as R
73+import qualified Hedgehog.Gen as Gen
74+import qualified Hedgehog.Range as Range
75+import Roboservant.Hedgehog (elementOrFail)
76+import Roboservant.Types
85+import Servant (Endpoints, Proxy (Proxy), Server, ServerError)
86+import Type.Reflection (SomeTypeRep)
72data RoboservantException
88data RoboservantException
73− = RoboservantException
89+ = RoboservantException FailureType SomeException FuzzOp FuzzState
74− { failureReason :: FailureType,
75− serverException :: Maybe SomeException,
76− fuzzState :: FuzzState
78 deriving (Show)
90 deriving (Show)
79−
91+-- we believe in nussink, lebowski
80instance Exception RoboservantException
92instance Exception RoboservantException
82data FailureType
94data FailureType
83 = ServerCrashed
95 = ServerCrashed
84 | CheckerFailed
96 | CheckerFailed
85 | NoPossibleMoves
97 | NoPossibleMoves
86− | InsufficientCoverage Double
98+ deriving (Show,Eq)
87− deriving (Show, Eq)
99+
88−
100+-- update :: Callback Op (Opaque (NEL.NonEmpty Dynamic)) State
89−data FuzzOp
101+-- update = Update $ \s@State {..} op o' ->
90− = FuzzOp
102+-- case op of
91− { apiOffset :: ApiOffset,
103+-- Chewable offset tr v -> case _extract offset trs chewable of
92− provenance :: [Provenance]
104+-- Nothing -> error "internal error"
93− }
105+-- Just (tr, _extracted, rest) ->
108+-- -- so, what's going on here... we need to be able to grab and merge
109+-- -- a list of (typeref, var). this means that we need to be able to pull
110+-- -- out typereps symbolically too, we can't just rely on the dynamic being there.
111+-- -- Map.insertWith (<>) (dynTypeRep v) _ (pure o') stateRefs,
112+-- Map.insertWith (<>) tr (_ o') stateRefs,
115+-- (Op (ApiOffset _offset) _args) ->
117+-- { chewable = _ o' : chewable
126+data Reference = Reference
129+ } deriving (Show,Eq)
131+data FuzzOp = FuzzOp ApiOffset [Reference]
94− deriving (Show, Eq)
132+ deriving (Show,Eq)
96−data FuzzState
134+data Config
97− = FuzzState
135+ = Config
98− { path :: [FuzzOp],
136+ { seed :: [Dynamic]
99− stash :: Stash,
137+ , maxRuntime :: Int -- seconds to test for
100− currentRng :: StdGen
139+ , currentRng :: StdGen
103−
142+data FuzzState = FuzzState
104−data EndpointOption
143+ { path :: [FuzzOp]
105− = forall as.
144+ , stash :: Map TypeRep (NEL.NonEmpty (Reference,Dynamic))
106− (V.RecordToList as, V.RMap as) =>
108− { eoCall :: V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))),
109− eoArgs :: V.Rec (TypedF StashValue) as
119− { textual :: String,
120− rsException :: RoboservantException
122 deriving (Show)
146 deriving (Show)
148+fuzz :: forall api. (FlattenServer api, ToReifiedApi (Endpoints api))
153+fuzz server Config{..} checker =
154+ -- either we time out without finding an error, which is fine, or we find an error
155+ -- and throw an exception that propagates through this.
156+ void $ timeout (maxRuntime * 1000000) ( execStateT (forever go) FuzzState{..})
126−-- fuzzClient :: Client api -> Config -> IO (Maybe Report)
127−-- fuzzClient = undefined
135−fuzz' reifiedApi Config {..} = handle (pure . Just . formatException) $ do
137− stash = addToStash seed mempty
138− currentRng = mkStdGen rngSeed
139− deadline :: UTCTime <- addUTCTime (realToFrac $ maxRuntime * 1000000) <$> getCurrentTime
140− (stopreason, _fs) <-
142− (untilDone (maxReps, deadline) go <* (evaluateCoverage =<< get))
144− logInfo $ show stopreason
147− -- something less terrible later
148− formatException :: RoboservantException -> Report
149− formatException r@(RoboservantException failureType exception _state) =
151− (unlines [show failureType, show exception])
153− displayDiagnostics FuzzState {..} = liftIO $
155− ["api endpoints covered"]
156− <> (map show . Set.toList . Set.fromList $ map apiOffset path)
157− <> ["", "types in stash"]
158− <> DM.foldrWithKey (\_ v r -> (show . NEL.length . getStashValue $ v) : r) [] (getStash stash)
159− -- <> (map (show . NEL.length . getStashValue ) $ DM.assocs (getStash stash))
161− -- (show . NEL.length $ getStashValue v))
163− evaluateCoverage f@FuzzState {..}
160+ reifiedApi = toReifiedApi (flattenServer @api server) (Proxy @(Endpoints api))
164− | coverage > coverageThreshold = pure ()
161+
165− | otherwise = do
162+ elementOrFail :: MonadState FuzzState m
166− displayDiagnostics f
163+ => [a] -> m a
167− throw $ RoboservantException (InsufficientCoverage coverage) Nothing f
164+ elementOrFail = undefined
169− hitRoutes = fromIntegral . Set.size . Set.fromList $ map apiOffset path
170− totalRoutes = fromIntegral routeCount
171− coverage = hitRoutes / totalRoutes
172− untilDone :: MonadIO m => (Integer, UTCTime) -> m a -> m StopReason
173− untilDone (0, _) _ = pure HitMaxIterations
174− untilDone (n, deadline) action = do
175− now <- liftIO getCurrentTime
180− untilDone (n -1, deadline) action
182− routeCount = length reifiedApi
184− (MonadState FuzzState m, MonadIO m) =>
166+ genOp :: MonadState FuzzState m
185− [a] ->
167+ => m FuzzOp
186− m a
168+ genOp = do -- fs@FuzzState{..} = do
187− elementOrFail [] = liftIO . throw . RoboservantException NoPossibleMoves Nothing =<< get
188− elementOrFail l = do
190− let (index, newGen) = randomR (0, length l - 1) (currentRng st)
191− modify' $ \st' -> st' {currentRng = newGen}
194− (MonadState FuzzState m, MonadIO m) =>
196− (V.RecordToList as, V.RMap as) =>
198− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
199− V.Rec (TypedF V.Identity) as ->
203− withOp callback = do
204 -- choose a call to make, from the endpoints with fillable arguments.
169 -- choose a call to make, from the endpoints with fillable arguments.
205− (offset, EndpointOption {..}) <- elementOrFail . options =<< get
170+ (offset, args) <- elementOrFail . options =<< get
206− r <-
171+ -- choose a (symbolic) argument from each. we drop the Dynamic here because it's
207− V.rtraverse
172+ -- not showable, which is no good for tests - though possibly it would make sense
208− ( \(tr :*: StashValue svs _) ->
173+ -- to use the constrained-dynamic stuff here? (or even just stash the string form)
209− elementOrFail $
174+ r <- (FuzzOp offset) <$> mapM (elementOrFail . fmap fst . NEL.toList) args
211− (\i xy -> V.Const i :*: tr :*: xy)
219− (\(V.Const index :*: tr :*: _) -> Provenance (R.SomeTypeRep tr) index)
223− (\(_ :*: tr :*: (_, x)) -> tr :*: V.Identity x)
225− modify' (\f -> f {path = path f <> [pathSegment]})
175+ modify' (\f -> f { path = r:path f })
226− callback pathSegment eoCall argValues
176+ pure r
179+-- chooseOne opts = do
181+-- (offset,) <$> mapM elementOrFail args
228− options :: FuzzState -> [(ApiOffset, EndpointOption)]
183+ options :: FuzzState -> [(ApiOffset, [NEL.NonEmpty (Reference, Dynamic)])]
229− options FuzzState {..} =
184+ options FuzzState{..} =
231− ( \(offset, ReifiedEndpoint {..}) -> do
186+ ( \(offset, (argreps, _dynCall)) -> (offset,) <$> do
232− args <- V.rtraverse (\(tr :*: Argument bf) -> (tr :*:) <$> bf stash) reArguments
187+ mapM (flip Map.lookup stash) argreps
233− pure (offset, EndpointOption reEndpointFunc args)
235 reifiedApi
189 reifiedApi
237− (MonadState FuzzState m, MonadIO m, V.RecordToList as, V.RMap as) =>
190+ execute :: MonadState FuzzState m
238− FuzzOp ->
191+ => FuzzOp -> m ()
239− V.Curried as (IO (Either InteractionError (NonEmpty (Dynamic, Int)))) ->
192+ execute = undefined
240− V.Rec (TypedF V.Identity) as ->
193+
242− execute fuzzop func args = do
195+
243− (liftIO . logInfo . show . (fuzzop,) . stash) =<< get
196+ -- execute (Op (ApiOffset offset) args) = do
244− liftIO (V.runcurry' func argVals) >>= \case
197+ -- fmap Opaque . liftIO $ do
245− Left (e::InteractionError) ->
198+ -- let realArgs = map opaque args
246− if fatalError e
199+ -- let (_offset, (_staticArgs, endpoint)) = staticRoutes !! offset
247− then throw e
200+ -- -- now, magic happens: we apply some dynamic arguments to a dynamic
248− else pure ()
201+ -- -- function and hopefully something useful pops out the end.
202+ -- func = foldr (\arg curr -> flip dynApply arg =<< curr) (Just endpoint) realArgs
203+ -- let showable = "blah" -- map dynTypeRep (_endpoint : realArgs)
205+ -- Nothing -> error ("all screwed up: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
207+ -- case fromDynamic f' of
208+ -- Nothing -> error ("all screwed up: " <> maybe ("nothing: " <> show showable) (show . dynTypeRep) func)
209+ -- Just f -> liftIO f >>= \case
210+ -- Left (serverError :: ServerError) -> error (show serverError)
249− Right (dyn :: NEL.NonEmpty (Dynamic, Int)) ->
211+ -- Right (_typeRep :: SomeTypeRep, (dyn :: NEL.NonEmpty Dynamic)) -> pure dyn
251− ( \fs@FuzzState {..} ->
213+
252− fs {stash = addToStash (NEL.toList dyn) stash}
255− argVals = V.rmap (\(_ :*: V.Identity x) -> V.Identity x) args
256− -- argTypes = recordToList' (\(tr :*: _) -> R.SomeTypeRep tr) args
258− (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m) =>
214+ go :: (MonadState FuzzState m, MonadIO m, MonadBaseControl IO m)
260− go = withOp $ \op func args -> do
216+ go = do
261− catches
217+ fuzzOp <- genOp
262− (execute op func args)
218+ catch (execute fuzzOp)
263− [ Handler (\(e :: SomeAsyncException) -> throw e),
265− ( \(e :: SomeException) ->
266− throw . RoboservantException ServerCrashed (Just e) =<< get
219+ (\(e :: SomeException) -> throw . RoboservantException ServerCrashed e fuzzOp =<< get)
269− catch
220+ catch (liftIO checker)
270− (liftIO healthCheck)
271− (\(e :: SomeException) -> throw . RoboservantException CheckerFailed (Just e) =<< get)
221+ (\(e :: SomeException) -> throw . RoboservantException CheckerFailed e fuzzOp =<< get)
273−addToStash ::
223+ -- actions <-
274− [(Dynamic, Int)] ->
224+ -- forAll $ do
275− Stash ->
225+ -- Gen.sequential
276− Stash
226+ -- (Range.linear 1 100)
277−addToStash result stash =
227+ -- emptyState
278− foldr
228+ -- (fmap preload seed <> [callEndpoint reifiedApi])
279− ( \(Dynamic tr x, hashed) (Stash dict) ->
229+ -- executeSequential emptyState actions
284− (StashValue (([Provenance (R.SomeTypeRep tr) 0], x) :| []) (IntSet.singleton hashed))
294− renumber (StashValue singleDyn singleHash) orig@(StashValue l intSet)
295− | not $ IntSet.null (singleHash `IntSet.intersection` intSet) = orig
298− ( case NEL.toList singleDyn of
299− [([Provenance tr _], dyn)] ->
300− l <> pure ([Provenance tr (length (NEL.last l) + 1)], dyn)
301− _ -> error "should be impossible"
303− (IntSet.union singleHash intSet)
305−-- why isn't this in vinyl?
307− (V.RecordToList as, V.RMap as) =>
308− (forall x. f x -> a) ->
311−recordToList' f = V.recordToList . V.rmap (V.Const . f)