haskell/ghcide
1
1
Fork 0
mirror of https://github.com/haskell/ghcide.git synced 2024-10-05 17:48:19 +03:00
Code Issues Projects Releases Wiki Activity

Allow to easily customise the example used for benchmarks (#838)

Browse Source 
* [ghcide-bench] allow custom example

* [bench] allow custom example

* Add v0.4.0 entry for completeness

* Rename benchmark artifacts

bench/hist.yaml --> bench/config.yaml

bench-hist --> bench-results

* Fix Cabal file

* Fix tests

* No need for hardcoded experiment positions
This commit is contained in:
Pepe Iborra 2020-09-29 07:47:09 +01:00 committed by GitHub
parent 85f3738e82
commit a52741838b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 247 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.azure/linux-bench.yml
View File

@ -42,7 +42,7 @@ jobs:
stack bench --ghc-options=-Werror --stack-yaml=$STACK_YAML || stack bench --ghc-options=-Werror --stack-yaml=$STACK_YAML || stack bench --ghc-options=-Werror --stack-yaml=$STACK_YAML
displayName: 'stack bench --ghc-options=-Werror'
- bash: |
cat bench-hist/results.csv
cat bench-results/results.csv
displayName: "cat results"
- publish: bench-hist
artifact: benchmarks

2
.gitignore vendored
View File

@ -8,7 +8,7 @@ cabal.project.local
.vscode
/.hlint-*
bench/example/
bench-hist/
bench-results/
bench-temp/
.shake/
ghcide

4
bench/README.md
View File

@ -9,7 +9,7 @@ performance analysis of ghcide:
- Run with `stack bench` or `cabal bench`,
- Requires a `ghcide-bench` binary in the PATH (usually provided by stack/cabal),
- Calls `cabal` (or `stack`, configurable) internally to build the project,
- Driven by the `hist.yaml` configuration file.
- Driven by the `config.yaml` configuration file.
By default it compares HEAD with "master"
Further details available in the module header comments.
Further details available in the config file and the module header comments.

12
bench/hist.yaml → bench/config.yaml
View File

@ -8,7 +8,16 @@ buildTool: stack
ghcideBench: ghcide-bench
# Output folder for the experiments
outputFolder: bench-hist
outputFolder: bench-results
# Example project used to run the experiments
# Can either be a Hackage package (name,version)
# or a local project (path) with a valid `hie.yaml` file
example:
name: Cabal
version: 3.0.0.0
# path: path/to/example
module: Distribution/Simple.hs
# The set of experiments to execute
experiments:
@ -40,5 +49,6 @@ versions:
# - v0.1.0
# - v0.2.0
# - v0.3.0
# - v0.4.0
- upstream: origin/master
- HEAD

2
bench/exe/Main.hs
View File

@ -45,6 +45,6 @@ main = do
output "starting test"
cleanUp <- setup
SetupResult{..} <- setup
runBenchmarks experiments `finally` cleanUp

16
bench/hist/Main.hs
View File

@ -2,14 +2,14 @@
A Shake script to analyze the performance of ghcide over the git history of the project
Driven by a config file `bench/hist.yaml` containing the list of Git references to analyze.
Driven by a config file `bench/config.yaml` containing the list of Git references to analyze.
Builds each one of them and executes a set of experiments using the ghcide-bench suite.
The results of the benchmarks and the analysis are recorded in the file
system with the following structure:
bench-hist
bench-results
├── <git-reference> - one folder per version
│   ├── <experiment>.benchmark-gcStats - RTS -s output
│   ├── <experiment>.csv - stats for the experiment
@ -31,8 +31,8 @@
To build a specific analysis, enumerate the desired file artifacts
> stack bench --ba "bench-hist/HEAD/results.csv bench-hist/HEAD/edit.diff.svg"
> cabal bench --benchmark-options "bench-hist/HEAD/results.csv bench-hist/HEAD/edit.diff.svg"
> stack bench --ba "bench-results/HEAD/results.csv bench-results/HEAD/edit.diff.svg"
> cabal bench --benchmark-options "bench-results/HEAD/results.csv bench-results/HEAD/edit.diff.svg"
-}
{-# LANGUAGE DeriveAnyClass #-}
@ -49,6 +49,7 @@ import qualified Data.Text as T
import Data.Yaml ((.!=), (.:?), FromJSON (..), ToJSON (..), Value (..), decodeFileThrow)
import Development.Shake
import Development.Shake.Classes (Binary, Hashable, NFData)
import Experiments.Types (exampleToOptions, Example(..))
import GHC.Exts (IsList (..))
import GHC.Generics (Generic)
import qualified Graphics.Rendering.Chart.Backend.Diagrams as E
@ -61,7 +62,7 @@ import qualified Text.ParserCombinators.ReadP as P
import Text.Read (Read (..), get, readMaybe, readP_to_Prec)
config :: FilePath
config = "bench/hist.yaml"
config = "bench/config.yaml"
-- | Read the config without dependency
readConfigIO :: FilePath -> IO Config
@ -197,12 +198,12 @@ main = shakeArgs shakeOptions {shakeChange = ChangeModtimeAndDigest} $ do
"-v",
"--samples=" <> show samples,
"--csv=" <> outcsv,
"--example-package-version=3.0.0.0",
"--ghcide-options= +RTS -I0.5 -RTS",
"--ghcide=" <> ghcide,
"--select",
unescaped (unescapeExperiment (Escaped $ dropExtension exp))
] ++
exampleToOptions (example configStatic) ++
[ "--stack" | Stack == buildSystem]
cmd_ Shell $ "mv *.benchmark-gcStats " <> dropFileName outcsv
@ -281,6 +282,7 @@ findGhc Stack = do
data Config = Config
{ experiments :: [Unescaped String],
example :: Example,
samples :: Natural,
versions :: [GitCommit],
-- | Path to the ghcide-bench binary for the experiments
@ -290,7 +292,7 @@ data Config = Config
buildTool :: BuildSystem
}
deriving (Generic, Show)
deriving anyclass (FromJSON, ToJSON)
deriving anyclass (FromJSON)
data GitCommit = GitCommit
{ -- | A git hash, tag or branch name (e.g. v0.1.0)

298
bench/lib/Experiments.hs
View File

@ -1,6 +1,8 @@
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE ConstraintKinds #-}
{-# LANGUAGE ExistentialQuantification #-}
{-# LANGUAGE ImplicitParams #-}
{-# LANGUAGE ImpredicativeTypes #-}
module Experiments
( Bench(..)
@ -8,68 +10,67 @@ module Experiments
, Config(..)
, Verbosity(..)
, CabalStack(..)
, SetupResult(..)
, Example(..)
, experiments
, configP
, defConfig
, output
, setup
, runBench
, runBenchmarks
, exampleToOptions
) where
import Control.Applicative.Combinators (skipManyTill)
import Control.Concurrent
import Control.Exception.Safe
import Control.Monad.Extra
import Control.Monad.IO.Class
import Data.Aeson (Value(Null))
import Data.Char (isDigit)
import Data.List
import Data.Maybe
import qualified Data.Text as T
import Data.Version
import Development.IDE.Plugin.Test
import Experiments.Types
import Language.Haskell.LSP.Test
import Language.Haskell.LSP.Types
import Language.Haskell.LSP.Types.Capabilities
import Numeric.Natural
import Options.Applicative
import System.Directory
import System.Environment.Blank (getEnv)
import System.FilePath ((</>))
import System.Process
import System.Time.Extra
import Text.ParserCombinators.ReadP (readP_to_S)
import System.Environment.Blank (getEnv)
import Development.IDE.Plugin.Test
import Data.Aeson (Value(Null))
-- Points to a string in the target file,
-- convenient for hygienic edits
hygienicP :: Position
hygienicP = Position 854 23
hygienicEdit :: TextDocumentContentChangeEvent
hygienicEdit :: (?hygienicP :: Position) => TextDocumentContentChangeEvent
hygienicEdit =
TextDocumentContentChangeEvent
{ _range = Just (Range hygienicP hygienicP),
{ _range = Just (Range ?hygienicP ?hygienicP),
_rangeLength = Nothing,
_text = " "
}
breakingEdit :: TextDocumentContentChangeEvent
breakingEdit :: (?identifierP :: Position) => TextDocumentContentChangeEvent
breakingEdit =
TextDocumentContentChangeEvent
{ _range = Just (Range identifierP identifierP),
{ _range = Just (Range ?identifierP ?identifierP),
_rangeLength = Nothing,
_text = "a"
}
-- Points to the middle of an identifier,
-- convenient for requesting goto-def, hover and completions
identifierP :: Position
identifierP = Position 853 12
-- | Experiments have access to these special positions:
-- - hygienicP points to a string in the target file, convenient for hygienic edits
-- - identifierP points to the middle of an identifier, convenient for goto-def, hover and completions
type HasPositions = (?hygienicP :: Position, ?identifierP :: Position)
experiments :: [Bench]
experiments =
[ ---------------------------------------------------------------------------------------
bench "hover" 10 $ \doc ->
isJust <$> getHover doc identifierP,
isJust <$> getHover doc ?identifierP,
---------------------------------------------------------------------------------------
bench "edit" 10 $ \doc -> do
changeDoc doc [hygienicEdit]
@ -78,10 +79,10 @@ experiments =
---------------------------------------------------------------------------------------
bench "hover after edit" 10 $ \doc -> do
changeDoc doc [hygienicEdit]
isJust <$> getHover doc identifierP,
isJust <$> getHover doc ?identifierP,
---------------------------------------------------------------------------------------
bench "getDefinition" 10 $ \doc ->
not . null <$> getDefinitions doc identifierP,
not . null <$> getDefinitions doc ?identifierP,
---------------------------------------------------------------------------------------
bench "documentSymbols" 100 $
fmap (either (not . null) (not . null)) . getDocumentSymbols,
@ -92,7 +93,7 @@ experiments =
---------------------------------------------------------------------------------------
bench "completions after edit" 10 $ \doc -> do
changeDoc doc [hygienicEdit]
not . null <$> getCompletions doc identifierP,
not . null <$> getCompletions doc ?identifierP,
---------------------------------------------------------------------------------------
benchWithSetup
"code actions"
@ -100,7 +101,7 @@ experiments =
( \doc -> do
changeDoc doc [breakingEdit]
waitForProgressDone
return identifierP
return ?identifierP
)
( \p doc -> do
not . null <$> getCodeActions doc (Range p p)
@ -111,7 +112,7 @@ experiments =
10
( \doc -> do
changeDoc doc [breakingEdit]
return identifierP
return ?identifierP
)
( \p doc -> do
changeDoc doc [hygienicEdit]
@ -122,40 +123,12 @@ experiments =
---------------------------------------------------------------------------------------------
examplePackageName :: HasConfig => String
examplePackageName = name
where
(name, _, _) = examplePackageUsed ?config
examplePackage :: HasConfig => String
examplePackage = name <> "-" <> showVersion version
where
(name, version, _) = examplePackageUsed ?config
exampleModulePath :: HasConfig => FilePath
exampleModulePath = path
where
(_,_, path) = examplePackageUsed ?config
exampleModulePath = exampleModule (example ?config)
examplesPath :: FilePath
examplesPath = "bench/example"
data Verbosity = Quiet | Normal | All
deriving (Eq, Show)
data Config = Config
{ verbosity :: !Verbosity,
-- For some reason, the Shake profile files are truncated and won't load
shakeProfiling :: !(Maybe FilePath),
outputCSV :: !FilePath,
buildTool :: !CabalStack,
ghcideOptions :: ![String],
matches :: ![String],
repetitions :: Maybe Natural,
ghcide :: FilePath,
timeoutLsp :: Int,
examplePackageUsed :: (String, Version, String)
}
deriving (Eq, Show)
defConfig :: Config
Success defConfig = execParserPure defaultPrefs (info configP fullDesc) []
@ -164,9 +137,6 @@ quiet, verbose :: Config -> Bool
verbose = (== All) . verbosity
quiet = (== Quiet) . verbosity
data CabalStack = Cabal | Stack
deriving (Eq, Show)
type HasConfig = (?config :: Config)
configP :: Parser Config
@ -184,9 +154,15 @@ configP =
<*> optional (option auto (long "samples" <> metavar "NAT" <> help "override sampling count"))
<*> strOption (long "ghcide" <> metavar "PATH" <> help "path to ghcide" <> value "ghcide")
<*> option auto (long "timeout" <> value 60 <> help "timeout for waiting for a ghcide response")
<*> ( (,,) <$> strOption (long "example-package-name" <> value "Cabal")
<*> ( GetPackage <$> strOption (long "example-package-name" <> value "Cabal")
<*> moduleOption
<*> option versionP (long "example-package-version" <> value (makeVersion [3,2,0,0]))
<*> strOption (long "example-package-module" <> metavar "PATH" <> value "Distribution/Simple.hs"))
<|>
UsePackage <$> strOption (long "example-path")
<*> moduleOption
)
where
moduleOption = strOption (long "example-module" <> metavar "PATH" <> value "Distribution/Simple.hs")
versionP :: ReadM Version
versionP = maybeReader $ extract . readP_to_S parseVersion
@ -205,8 +181,8 @@ data Bench = forall setup.
{ name :: !String,
enabled :: !Bool,
samples :: !Natural,
benchSetup :: TextDocumentIdentifier -> Session setup,
experiment :: setup -> Experiment
benchSetup :: HasPositions => TextDocumentIdentifier -> Session setup,
experiment :: HasPositions => setup -> Experiment
}
select :: HasConfig => Bench -> Bool
@ -218,26 +194,26 @@ select Bench {name, enabled} =
benchWithSetup ::
String ->
Natural ->
(TextDocumentIdentifier -> Session p) ->
(p -> Experiment) ->
(HasPositions => TextDocumentIdentifier -> Session p) ->
(HasPositions => p -> Experiment) ->
Bench
benchWithSetup name samples benchSetup experiment = Bench {..}
where
enabled = True
bench :: String -> Natural -> Experiment -> Bench
bench :: String -> Natural -> (HasPositions => Experiment) -> Bench
bench name defSamples userExperiment =
benchWithSetup name defSamples (const $ pure ()) experiment
where
experiment () = userExperiment
runBenchmarks :: HasConfig => [Bench] -> IO ()
runBenchmarks allBenchmarks = do
runBenchmarksFun :: HasConfig => FilePath -> [Bench] -> IO ()
runBenchmarksFun dir allBenchmarks = do
let benchmarks = [ b{samples = fromMaybe (samples b) (repetitions ?config) }
| b <- allBenchmarks
, select b ]
results <- forM benchmarks $ \b@Bench{name} ->
let run dir = runSessionWithConfig conf (cmd name dir) lspTestCaps dir
let run = runSessionWithConfig conf (cmd name dir) lspTestCaps dir
in (b,) <$> runBench run b
-- output raw data as CSV
@ -338,96 +314,138 @@ waitForProgressDone :: Session ()
waitForProgressDone =
void(skipManyTill anyMessage message :: Session WorkDoneProgressEndNotification)
runBench :: (?config::Config) => (String -> Session BenchRun -> IO BenchRun) -> Bench -> IO BenchRun
runBench runSess Bench {..} = handleAny (\e -> print e >> return badRun)
$ runSess dir
runBench ::
(?config :: Config) =>
(Session BenchRun -> IO BenchRun) ->
(HasPositions => Bench) ->
IO BenchRun
runBench runSess b = handleAny (\e -> print e >> return badRun)
$ runSess
$ do
doc <- openDoc exampleModulePath "haskell"
(startup, _) <- duration $ do
waitForProgressDone
-- wait again, as the progress is restarted once while loading the cradle
-- make an edit, to ensure this doesn't block
changeDoc doc [hygienicEdit]
waitForProgressDone
liftIO $ output $ "Running " <> name <> " benchmark"
(runSetup, userState) <- duration $ benchSetup doc
let loop !userWaits !delayedWork 0 = return $ Just (userWaits, delayedWork)
loop !userWaits !delayedWork n = do
(t, res) <- duration $ experiment userState doc
if not res
then return Nothing
else do
output (showDuration t)
-- Wait for the delayed actions to finish
waitId <- sendRequest (CustomClientMethod "test") WaitForShakeQueue
(td, resp) <- duration $ skipManyTill anyMessage $ responseForId waitId
case resp of
ResponseMessage{_result=Right Null} -> do
loop (userWaits+t) (delayedWork+td) (n -1)
_ ->
-- Assume a ghcide build lacking the WaitForShakeQueue command
loop (userWaits+t) delayedWork (n -1)
-- Setup the special positions used by the experiments
lastLine <- length . T.lines <$> documentContents doc
changeDoc doc [TextDocumentContentChangeEvent
{ _range = Just (Range (Position lastLine 0) (Position lastLine 0))
, _rangeLength = Nothing
, _text = T.unlines
[ "_hygienic = \"hygienic\""
, "_identifier = _hygienic"
]
}]
let
-- Points to a string in the target file,
-- convenient for hygienic edits
?hygienicP = Position lastLine 15
let
-- Points to the middle of an identifier,
-- convenient for requesting goto-def, hover and completions
?identifierP = Position (lastLine+1) 15
(runExperiment, result) <- duration $ loop 0 0 samples
let success = isJust result
(userWaits, delayedWork) = fromMaybe (0,0) result
case b of
Bench{..} -> do
(startup, _) <- duration $ do
waitForProgressDone
-- wait again, as the progress is restarted once while loading the cradle
-- make an edit, to ensure this doesn't block
changeDoc doc [hygienicEdit]
waitForProgressDone
-- sleep to give ghcide a chance to GC
liftIO $ threadDelay 1100000
liftIO $ output $ "Running " <> name <> " benchmark"
(runSetup, userState) <- duration $ benchSetup doc
let loop !userWaits !delayedWork 0 = return $ Just (userWaits, delayedWork)
loop !userWaits !delayedWork n = do
(t, res) <- duration $ experiment userState doc
if not res
then return Nothing
else do
output (showDuration t)
-- Wait for the delayed actions to finish
waitId <- sendRequest (CustomClientMethod "test") WaitForShakeQueue
(td, resp) <- duration $ skipManyTill anyMessage $ responseForId waitId
case resp of
ResponseMessage{_result=Right Null} -> do
loop (userWaits+t) (delayedWork+td) (n -1)
_ ->
-- Assume a ghcide build lacking the WaitForShakeQueue command
loop (userWaits+t) delayedWork (n -1)
(maxResidency, allocations) <- liftIO $
ifM (doesFileExist gcStats)
(parseMaxResidencyAndAllocations <$> readFile gcStats)
(pure (0,0))
(runExperiment, result) <- duration $ loop 0 0 samples
let success = isJust result
(userWaits, delayedWork) = fromMaybe (0,0) result
gcStats = escapeSpaces (name <> ".benchmark-gcStats")
return BenchRun {..}
where
dir = "bench/example/" <> examplePackage
gcStats = escapeSpaces (name <> ".benchmark-gcStats")
-- sleep to give ghcide a chance to GC
liftIO $ threadDelay 1100000
setup :: HasConfig => IO (IO ())
(maxResidency, allocations) <- liftIO $
ifM (doesFileExist gcStats)
(parseMaxResidencyAndAllocations <$> readFile gcStats)
(pure (0,0))
return BenchRun {..}
data SetupResult = SetupResult {
runBenchmarks :: [Bench] -> IO (),
-- | Path to the setup benchmark example
benchDir :: FilePath,
cleanUp :: IO ()
}
setup :: HasConfig => IO SetupResult
setup = do
alreadyExists <- doesDirectoryExist examplesPath
when alreadyExists $ removeDirectoryRecursive examplesPath
let path = examplesPath </> examplePackage
case buildTool ?config of
Cabal -> do
callCommand $ "cabal get -v0 " <> examplePackage <> " -d " <> examplesPath
writeFile
(path </> "hie.yaml")
("cradle: {cabal: {component: " <> show examplePackageName <> "}}")
-- Need this in case there is a parent cabal.project somewhere
writeFile
(path </> "cabal.project")
"packages: ."
writeFile
(path </> "cabal.project.local")
""
Stack -> do
callCommand $ "stack --silent unpack " <> examplePackage <> " --to " <> examplesPath
-- Generate the stack descriptor to match the one used to build ghcide
stack_yaml <- fromMaybe "stack.yaml" <$> getEnv "STACK_YAML"
stack_yaml_lines <- lines <$> readFile stack_yaml
writeFile (path </> stack_yaml)
(unlines $
"packages: [.]" :
[ l
| l <- stack_yaml_lines
, any (`isPrefixOf` l)
["resolver"
,"allow-newer"
,"compiler"]
]
)
benchDir <- case example ?config of
UsePackage{..} -> return examplePath
GetPackage{..} -> do
let path = examplesPath </> package
package = exampleName <> "-" <> showVersion exampleVersion
case buildTool ?config of
Cabal -> do
callCommand $ "cabal get -v0 " <> package <> " -d " <> examplesPath
writeFile
(path </> "hie.yaml")
("cradle: {cabal: {component: " <> exampleName <> "}}")
-- Need this in case there is a parent cabal.project somewhere
writeFile
(path </> "cabal.project")
"packages: ."
writeFile
(path </> "cabal.project.local")
""
Stack -> do
callCommand $ "stack --silent unpack " <> package <> " --to " <> examplesPath
-- Generate the stack descriptor to match the one used to build ghcide
stack_yaml <- fromMaybe "stack.yaml" <$> getEnv "STACK_YAML"
stack_yaml_lines <- lines <$> readFile stack_yaml
writeFile (path </> stack_yaml)
(unlines $
"packages: [.]" :
[ l
| l <- stack_yaml_lines
, any (`isPrefixOf` l)
["resolver"
,"allow-newer"
,"compiler"]
]
)
writeFile
(path </> "hie.yaml")
("cradle: {stack: {component: " <> show (examplePackageName <> ":lib") <> "}}")
writeFile
(path </> "hie.yaml")
("cradle: {stack: {component: " <> show (exampleName <> ":lib") <> "}}")
return path
whenJust (shakeProfiling ?config) $ createDirectoryIfMissing True
return $ removeDirectoryRecursive examplesPath
let cleanUp = case example ?config of
GetPackage{} -> removeDirectoryRecursive examplesPath
UsePackage{} -> return ()
runBenchmarks = runBenchmarksFun benchDir
return SetupResult{..}
--------------------------------------------------------------------------------------------

54
bench/lib/Experiments/Types.hs Normal file
View File

@ -0,0 +1,54 @@
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DerivingStrategies #-}
module Experiments.Types where
import Data.Aeson
import Data.Version
import Numeric.Natural
data CabalStack = Cabal | Stack
deriving (Eq, Show)
data Verbosity = Quiet | Normal | All
deriving (Eq, Show)
data Config = Config
{ verbosity :: !Verbosity,
-- For some reason, the Shake profile files are truncated and won't load
shakeProfiling :: !(Maybe FilePath),
outputCSV :: !FilePath,
buildTool :: !CabalStack,
ghcideOptions :: ![String],
matches :: ![String],
repetitions :: Maybe Natural,
ghcide :: FilePath,
timeoutLsp :: Int,
example :: Example
}
deriving (Eq, Show)
data Example
= GetPackage {exampleName, exampleModule :: String, exampleVersion :: Version}
| UsePackage {examplePath :: FilePath, exampleModule :: String}
deriving (Eq, Show)
instance FromJSON Example where
parseJSON = withObject "example" $ \x -> do
exampleModule <- x .: "module"
path <- x .:? "path"
case path of
Just examplePath -> return UsePackage{..}
Nothing -> do
exampleName <- x .: "name"
exampleVersion <- x .: "version"
return GetPackage {..}
exampleToOptions :: Example -> [String]
exampleToOptions GetPackage{..} =
["--example-package-name", exampleName
,"--example-package-version", showVersion exampleVersion
,"--example-module", exampleModule
]
exampleToOptions UsePackage{..} =
["--example-path", examplePath
,"--example-module", exampleModule
]

9
ghcide.cabal
View File

@ -216,7 +216,9 @@ benchmark benchHist
type: exitcode-stdio-1.0
default-language: Haskell2010
ghc-options: -Wall -Wno-name-shadowing -threaded
main-is: bench/hist/Main.hs
main-is: Main.hs
hs-source-dirs: bench/hist bench/lib
other-modules: Experiments.Types
build-tool-depends:
ghcide:ghcide,
ghcide:ghcide-bench
@ -363,6 +365,7 @@ test-suite ghcide-tests
Development.IDE.Test
Development.IDE.Test.Runfiles
Experiments
Experiments.Types
default-extensions:
BangPatterns
DeriveFunctor
@ -395,13 +398,15 @@ executable ghcide-bench
lsp-test >= 0.11.0.2 && < 0.12,
optparse-applicative,
process,
safe-exceptions
safe-exceptions,
text
hs-source-dirs: bench/lib bench/exe
include-dirs: include
ghc-options: -threaded -Wall -Wno-name-shadowing
main-is: Main.hs
other-modules:
Experiments
Experiments.Types
default-extensions:
BangPatterns
DeriveFunctor

5
test/exe/Main.hs
View File

@ -3378,9 +3378,10 @@ benchmarkTests =
, Bench.repetitions = Just 3
, Bench.buildTool = Bench.Stack
} in
withResource Bench.setup id $ \_ -> testGroup "benchmark experiments"
withResource Bench.setup Bench.cleanUp $ \getResource -> testGroup "benchmark experiments"
[ expectFailCabal "Requires stack" $ testCase (Bench.name e) $ do
res <- Bench.runBench runInDir e
Bench.SetupResult{Bench.benchDir} <- getResource
res <- Bench.runBench (runInDir benchDir) e
assertBool "did not successfully complete 5 repetitions" $ Bench.success res
| e <- Bench.experiments
, Bench.name e /= "edit" -- the edit experiment does not ever fail