on master help sign in
Port the test infrastructure to Tasty

(This is WIP) [ci skip] `runtest.hs` is replaced by `TestRun.hs`. This program imports `TestData.hs` which holds the tests metadata, then turns it into tests usable by Tasty. The functional changes are the following: - The tasty framework provides a pretty output by default (colors, individual timings, proper indentation). Tests can run in parallel but the output is still deterministic. - Tasty.Golden does the heavy lifting, i.e. comparing outputs and measuring time spent. - Output is captured through a pipe connected to a bash process running the `run` script for each test ; this is implemented by the `runTest` function - Tests are organised in families, reflecting directories' prefixes. They are listed in `testFamiliesData` in `TestData.hs`. - Each test must be registered in `testFamiliesData` under its own family. - A custom option has been implemented to run the test suite with the node code generator. The list of compatible code generators for each test is provided in `testFamiliesData`. - Tasty.Rerun provides rudimentary incremental testing. The cache is in the `.tasty-rerun-log` file, which is therefore added to `.gitignore`. The previous Makefile in `test` has disappeared in favour of passing arguments to the test program. Note that this commit disables testing because a compatibility layer for `cabal test` must be implemented first. This is necessary because the test program requires external dependencies. However, you should be able to run the tests with `stack test`.

commit54b94b9184064269a630c1183979e64ecc444b7f authorgpyh <yacine.hmito@gmail.com> dateWed 27 Jul 2016, 14:46 (10 years ago) parent4e69658
7 files changed+445−270
.gitignore+1 Makefile+3−3 idris.cabal+13−3 test/Makefile−34 test/TestData.hs+300 test/TestRun.hs+128 test/runtest.hs−230
.gitignore+1
@@ -22,6 +22,7 @@ test/*[0-9][0-9][0-9]/output
2222test/*[0-9][0-9][0-9]/*.exe
2323test/runtest.exe
2424test/runtest
25+.tasty-rerun-log
2526
2627# Haskell build tools
2728cabal-dev/
Makefile+3−3
@@ -20,13 +20,13 @@ build: dist/setup-config
2020test: doc test_c
2121
2222test_c:
23− $(MAKE) -C test IDRIS=../dist/build/idris/idris test
23+ echo "test_c not supported"
2424
2525test_js:
26− $(MAKE) -C test IDRIS=../dist/build/idris/idris test_js
26+ echo "test_js not supported"
2727
2828test_timed:
29− $(MAKE) -C test IDRIS=../dist/build/idris/idris time
29+ echo "test_timed not supported"
3030
3131lib_clean:
3232 $(MAKE) -C libs IDRIS=../../dist/build/idris/idris RTS=../../dist/build/rts/libidris_rts clean
idris.cabal+13−3
@@ -156,8 +156,8 @@ Extra-source-files:
156156 libs/pruviloj/Pruviloj/Derive/*.idr
157157 libs/pruviloj/Pruviloj/Internals/*.idr
158158
159− test/Makefile
160− test/runtest.hs
159+ test/TestRun.hs
160+ test/TestData.hs
161161
162162 test/regression001/run
163163 test/regression001/expected
@@ -1158,7 +1158,8 @@ Executable idris
11581158
11591159Test-suite regression-and-sanity-tests
11601160 Type: exitcode-stdio-1.0
1161− Main-is: runtest.hs
1161+ Main-is: TestRun.hs
1162+ Other-modules: TestData
11621163 hs-source-dirs: test
11631164
11641165 Build-depends: idris
@@ -1169,7 +1170,16 @@ Test-suite regression-and-sanity-tests
11691170 , filepath
11701171 , directory
11711172 , haskeline >= 0.7
1173+ , optparse-applicative >= 0.11 && < 0.13
1174+ , tagged
1175+ , tasty >= 0.8
1176+ , tasty-golden >= 2.0
1177+ , tasty-rerun >= 1.0.0
1178+ , bytestring
11721179 , transformers
1180+ if impl(ghc < 7.10)
1181+ Extensions: DeriveDataTypeable
1182+
11731183
11741184 ghc-prof-options: -auto-all -caf-all
11751185 ghc-options: -threaded -rtsopts -funbox-strict-fields
test/Makefiledeleted−34
@@ -1,34 +0,0 @@
1−.PHONY: test test_js time update diff distclean $(TESTS)
2−
3−TESTS = $(sort $(patsubst %/,%.test,$(wildcard */)))
4−
5−test: $(TESTS)
6−
7−
8−info: runtest
9− @./runtest all
10−
11−%.test: runtest
12− @./runtest $(patsubst %.test,%,$@) -q
13−
14−test_js: runtest
15− @./runtest without tutorial007 sugar004 reg029 reg052 io001 dsl002 io003 effects001 effects002 basic007 basic011 ffi006 ffi007 ffi008 primitives005 primitives006 views003 opts --codegen node
16−
17−update: runtest
18− @./runtest all -u
19−
20−diff: runtest
21− @./runtest all -d
22−
23−time: runtest
24− @./runtest all -t
25−
26−distclean:
27− @rm runtest
28− @rm -f *~
29− @rm -f */output
30−
31−
32−runtest:
33− @ghc --make runtest.hs
34− @rm runtest.o runtest.hi
test/TestData.hsnew file+300
@@ -0,0 +1,300 @@
1+module TestData where
2+
3+import Data.IntMap as IMap
4+import Data.Map.Strict as Map
5+import Data.Set as Set
6+
7+data Codegen = C | JS deriving (Show, Eq, Ord)
8+type Index = Int
9+data CompatCodegen = ANY | C_CG | NODE_CG | NONE
10+
11+-- A TestFamily groups tests that share the same theme
12+data TestFamily = TestFamily {
13+ -- A shorter lowcase name to use in filenames
14+ id :: String,
15+ -- A proper name for the test family that will be displayed
16+ name :: String,
17+ -- A map of test metadata:
18+ -- - The key is the index (>=1 && <1000)
19+ -- - The value is the set of compatible code generators,
20+ -- or Nothing if the test doesn't depend on a code generator
21+ tests :: IntMap (Maybe (Set Codegen))
22+} deriving (Show)
23+
24+toCodegenSet :: CompatCodegen -> Maybe (Set Codegen)
25+toCodegenSet compatCodegen = fmap Set.fromList mList where
26+ mList = case compatCodegen of
27+ ANY -> Just [ C, JS ]
28+ C_CG -> Just [ C ]
29+ NODE_CG -> Just [ JS ]
30+ NONE -> Nothing
31+
32+testFamilies :: [TestFamily]
33+testFamilies = fmap instanciate testFamiliesData where
34+ instanciate (id, name, testsData) = TestFamily id name tests where
35+ tests = IMap.fromList (fmap makeSetCodegen testsData)
36+ makeSetCodegen (index, codegens) = (index, toCodegenSet codegens)
37+
38+testFamiliesForCodegen :: Codegen -> [TestFamily]
39+testFamiliesForCodegen codegen =
40+ fmap (\testFamily -> testFamily {tests = IMap.filter f (tests testFamily)})
41+ testFamilies
42+ where
43+ f mCodegens = case mCodegens of
44+ Just codegens -> Set.member codegen codegens
45+ Nothing -> True
46+
47+-- The data to instanciate testFamilies
48+-- The first column is the id
49+-- The second column is the proper name (the prefix of the subfolders)
50+-- The third column is the data for each test
51+testFamiliesData :: [(String, String, [(Index, CompatCodegen)])]
52+testFamiliesData = [
53+ ("basic", "Basic",
54+ [ ( 1, ANY ),
55+ ( 2, ANY ),
56+ ( 3, ANY ),
57+ ( 4, ANY ),
58+ ( 5, ANY ),
59+ ( 6, ANY ),
60+ ( 7, C_CG ),
61+ ( 8, ANY ),
62+ ( 9, ANY ),
63+ ( 10, ANY ),
64+ ( 11, C_CG ),
65+ ( 12, ANY ),
66+ ( 13, ANY ),
67+ ( 14, ANY ),
68+ ( 15, ANY ),
69+ ( 16, ANY ),
70+ ( 17, ANY ),
71+ ( 18, ANY )]),
72+ ("bignum", "Bignum",
73+ [ ( 1, ANY ),
74+ ( 2, ANY )]),
75+ ("bounded", "Bounded",
76+ [ ( 1, ANY )]),
77+ ("corecords", "Corecords",
78+ [ ( 1, ANY ),
79+ ( 2, ANY )]),
80+ ("delab", "De-elaboration",
81+ [ ( 1, ANY )]),
82+ ("directives", "Directives",
83+ [ ( 1, ANY ),
84+ ( 2, ANY )]),
85+ ("disambig", "Disambiguation",
86+ [ ( 2, ANY )]),
87+ ("docs", "Documentation",
88+ [ ( 1, ANY ),
89+ ( 2, ANY ),
90+ ( 3, ANY ),
91+ ( 4, ANY ),
92+ ( 5, ANY )]),
93+ ("dsl", "DSL",
94+ [ ( 1, ANY ),
95+ ( 2, C_CG ),
96+ ( 3, ANY ),
97+ ( 4, ANY )]),
98+ ("effects", "Effects",
99+ [ ( 1, C_CG ),
100+ ( 2, C_CG ),
101+ ( 3, ANY ),
102+ ( 4, ANY ),
103+ ( 5, ANY )]),
104+ ("error", "Errors",
105+ [ ( 1, ANY ),
106+ ( 2, ANY ),
107+ ( 3, ANY ),
108+ ( 4, ANY ),
109+ ( 5, ANY ),
110+ ( 6, ANY ),
111+ ( 7, ANY ),
112+ ( 8, ANY )]),
113+ ("ffi", "FFI",
114+ [ ( 1, ANY ),
115+ ( 2, ANY ),
116+ ( 3, ANY ),
117+ ( 4, ANY ),
118+ ( 5, ANY ),
119+ ( 6, C_CG ),
120+ ( 7, C_CG ),
121+ ( 8, C_CG )]),
122+ ("folding", "Folding",
123+ [ ( 1, ANY )]),
124+ ("idrisdoc", "Idris documentation",
125+ [ ( 1, ANY ),
126+ ( 2, ANY ),
127+ ( 3, ANY ),
128+ ( 4, ANY ),
129+ ( 5, ANY ),
130+ ( 6, ANY ),
131+ ( 7, ANY ),
132+ ( 8, ANY ),
133+ ( 9, ANY )]),
134+ ("interactive", "Interactive editing",
135+ [ ( 1, ANY ),
136+ ( 2, ANY ),
137+ ( 3, ANY ),
138+ ( 4, ANY ),
139+ ( 5, ANY ),
140+ ( 6, ANY ),
141+ ( 7, ANY ),
142+ ( 8, ANY ),
143+ ( 9, ANY ),
144+ ( 10, ANY ),
145+ ( 11, ANY ),
146+ ( 12, ANY ),
147+ ( 13, ANY )]),
148+ ("interfaces", "Interfaces",
149+ [ ( 1, ANY ),
150+ ( 2, ANY ),
151+ ( 3, ANY ),
152+ ( 4, ANY ),
153+ ( 5, ANY ),
154+ ( 6, ANY )]),
155+ ("io", "IO monad",
156+ [ ( 1, C_CG ),
157+ ( 2, ANY ),
158+ ( 3, C_CG )]),
159+ ("literate", "Literate programming",
160+ [ ( 1, ANY )]),
161+ ("meta", "Meta-programming",
162+ [ ( 1, ANY ),
163+ ( 2, ANY ),
164+ ( 3, ANY ),
165+ ( 4, ANY )]),
166+ ("pkg", "Packages",
167+ [ ( 1, ANY ),
168+ ( 2, ANY ),
169+ ( 3, ANY ),
170+ ( 4, ANY )]),
171+ ("primitives", "Primitive types",
172+ [ ( 1, ANY ),
173+ ( 2, ANY ),
174+ ( 3, ANY ),
175+ ( 5, C_CG ),
176+ ( 6, C_CG )]),
177+ ("proof", "Theorem proving",
178+ [ ( 1, ANY ),
179+ ( 2, ANY ),
180+ ( 3, ANY ),
181+ ( 4, ANY ),
182+ ( 5, ANY ),
183+ ( 6, ANY ),
184+ ( 7, ANY ),
185+ ( 8, ANY ),
186+ ( 9, ANY ),
187+ ( 10, ANY ),
188+ ( 11, ANY )]),
189+ ("proofsearch", "Proof search",
190+ [ ( 1, ANY ),
191+ ( 2, ANY ),
192+ ( 3, ANY )]),
193+ ("pruviloj", "Pruviloj",
194+ [ ( 1, ANY )]),
195+ ("quasiquote", "Quasiquotations",
196+ [ ( 1, ANY ),
197+ ( 2, ANY ),
198+ ( 3, ANY ),
199+ ( 4, ANY ),
200+ ( 5, ANY ),
201+ ( 6, ANY )]),
202+ ("records", "Records",
203+ [ ( 1, ANY ),
204+ ( 2, ANY ),
205+ ( 3, ANY ),
206+ ( 4, ANY ),
207+ ( 5, ANY )]),
208+ ("reg", "Regressions",
209+ [ ( 2, ANY ),
210+ ( 3, ANY ),
211+ ( 4, ANY ),
212+ ( 5, ANY ),
213+ ( 6, ANY ),
214+ ( 7, ANY ),
215+ ( 10, ANY ),
216+ ( 13, ANY ),
217+ ( 16, ANY ),
218+ ( 17, ANY ),
219+ ( 18, ANY ),
220+ ( 20, ANY ),
221+ ( 23, ANY ),
222+ ( 24, ANY ),
223+ ( 25, ANY ),
224+ ( 27, ANY ),
225+ ( 28, ANY ),
226+ ( 29, C_CG ),
227+ ( 31, ANY ),
228+ ( 32, ANY ),
229+ ( 34, ANY ),
230+ ( 35, ANY ),
231+ ( 39, ANY ),
232+ ( 40, ANY ),
233+ ( 41, ANY ),
234+ ( 42, ANY ),
235+ ( 44, ANY ),
236+ ( 45, ANY ),
237+ ( 48, ANY ),
238+ ( 49, ANY ),
239+ ( 50, ANY ),
240+ ( 52, C_CG ),
241+ ( 54, ANY ),
242+ ( 55, ANY ),
243+ ( 56, ANY ),
244+ ( 67, ANY ),
245+ ( 68, ANY ),
246+ ( 69, ANY ),
247+ ( 70, ANY ),
248+ ( 72, ANY ),
249+ ( 75, ANY )]),
250+ ("regression", "Regression (loner)",
251+ [ ( 1 , ANY )]),
252+ ("sourceLocation", "Source location",
253+ [ ( 1 , ANY )]),
254+ ("sugar", "Syntactic sugar",
255+ [ ( 1, ANY ),
256+ ( 2, ANY ),
257+ ( 3, ANY ),
258+ ( 4, C_CG ),
259+ ( 5, ANY )]),
260+ ("syntax", "Syntax extensions",
261+ [ ( 1, ANY ),
262+ ( 2, ANY )]),
263+ ("tactics", "Tactics",
264+ [ ( 1, ANY )]),
265+ ("totality", "Totality checking",
266+ [ ( 1, ANY ),
267+ ( 2, ANY ),
268+ ( 3, ANY ),
269+ ( 4, ANY ),
270+ ( 5, ANY ),
271+ ( 6, ANY ),
272+ ( 7, ANY ),
273+ ( 8, ANY ),
274+ ( 9, ANY ),
275+ ( 10, ANY ),
276+ ( 11, ANY ),
277+ ( 12, ANY ),
278+ ( 13, ANY ),
279+ ( 14, ANY ),
280+ ( 15, ANY )]),
281+ ("tutorial", "Tutorial examples",
282+ [ ( 1, ANY ),
283+ ( 2, ANY ),
284+ ( 3, ANY ),
285+ ( 4, ANY ),
286+ ( 5, ANY ),
287+ ( 6, ANY ),
288+ ( 7, C_CG )]),
289+ ("unique", "Uniqueness types",
290+ [ ( 1, ANY ),
291+ ( 2, ANY ),
292+ ( 3, ANY )]),
293+ ("universes", "Universes",
294+ [ ( 1, ANY ),
295+ ( 2, ANY )]),
296+ ("views", "Views",
297+ [ ( 1, ANY ),
298+ ( 2, ANY ),
299+ ( 3, C_CG )])]
300+
test/TestRun.hsnew file+128
@@ -0,0 +1,128 @@
1+module Main where
2+
3+import Control.Monad
4+import Data.Typeable
5+import Data.Proxy
6+import Data.List
7+import Data.Map.Strict as Map
8+import Data.IntSet as ISet
9+import Data.IntMap as IMap
10+import qualified Data.ByteString as BS
11+import qualified Data.ByteString.Char8 as BSC
12+
13+import System.Directory
14+import System.Environment
15+import System.Process
16+import System.Info
17+import System.IO
18+import System.FilePath ((</>))
19+import Options.Applicative
20+import Test.Tasty
21+import Test.Tasty.Golden
22+import Test.Tasty.Golden.Advanced
23+import Test.Tasty.Runners
24+import Test.Tasty.Options
25+import Test.Tasty.Ingredients.Rerun
26+
27+import TestData
28+
29+--------------------------------------------------------------------- [ Config ]
30+
31+type Flags = [String]
32+
33+-- Add arguments to calls of idris executable
34+idrisFlags :: Flags
35+idrisFlags = []
36+
37+testDirectory :: String
38+testDirectory = "test"
39+
40+-------------------------------------------------------------------- [ Options ]
41+
42+-- The `--node` option makes idris use the node code generator
43+-- As a consequence, incompatible tests are removed
44+
45+newtype NodeOpt = NodeOpt Bool deriving (Eq, Ord, Typeable)
46+
47+nodeArg = "node"
48+nodeHelp = "Performs the tests with the node code generator"
49+instance IsOption NodeOpt where
50+ defaultValue = NodeOpt False
51+ parseValue = fmap NodeOpt . safeRead
52+ optionName = return nodeArg
53+ optionHelp = return nodeHelp
54+ optionCLParser = fmap NodeOpt $ switch (long nodeArg <> help nodeHelp)
55+
56+ingredients :: [Ingredient]
57+ingredients = [rerunningTests [consoleTestReporter],
58+ includingOptions [Option (Proxy :: Proxy NodeOpt)] ]
59+
60+----------------------------------------------------------------------- [ Core ]
61+
62+-- Compare a given file contents against the golden file contents
63+-- A ripoff of goldenVsFile from Tasty.Golden
64+test :: String -> String -> IO () -> TestTree
65+test name path act =
66+ goldenTest name (BS.readFile ref) (act >> BS.readFile new) cmp upd
67+ where
68+ ref = path </> "expected"
69+ new = path </> "output"
70+ cmp x y = return $ if x == y then Nothing
71+ else Just $ printDiff (ref, x) (new, y)
72+ upd = BS.writeFile ref
73+
74+-- Takes the filepath and content of `expected` and `output`
75+-- and formats an error message stating their difference
76+printDiff :: (String, BS.ByteString) -> (String, BS.ByteString) -> String
77+printDiff (ref, x) (new, y) =
78+ let refcnt = BSC.unpack x
79+ newcnt = BSC.unpack y
80+ printContent cnt =
81+ if Data.List.null cnt
82+ then " is empty...\n"
83+ else " is: \n" ++ unlines (fmap ((++) " ") (lines cnt))
84+ in
85+ "Test mismatch!\n" ++
86+ "Golden file " ++ ref ++ printContent refcnt ++
87+ "However, " ++ new ++ printContent newcnt
88+
89+-- Should always output a 3-charater string from a postive Int
90+indexToString :: Int -> String
91+indexToString index = let str = show index in
92+ (replicate (3 - length str) '0') ++ str
93+
94+-- Turns the collection of TestFamily into actual tests usable by Tasty
95+mkGoldenTests :: [TestFamily] -> Flags -> TestTree
96+mkGoldenTests testFamilies flags =
97+ testGroup "Regression and sanity tests"
98+ (fmap mkTestFamily testFamilies)
99+ where
100+ mkTestFamily (TestFamily id name tests) =
101+ testGroup name (fmap (mkTest id) (IMap.keys tests))
102+ mkTest id index =
103+ let testname = id ++ indexToString index
104+ path = testDirectory </> testname
105+ in
106+ test testname path (runTest path flags)
107+
108+-- Runs a test script
109+-- "bash" needed because Haskell has cmd as the default shell on windows, and
110+-- we also want to run the process with another current directory, so we get
111+-- this thing.
112+runTest :: String -> Flags -> IO ()
113+runTest path flags = do
114+ let run = (proc "bash" ("run" : flags)) {cwd = Just path,
115+ std_out = CreatePipe}
116+ (_, output, _) <- readCreateProcessWithExitCode run ""
117+ writeFile (path </> "output") output
118+
119+main :: IO ()
120+main = do
121+ defaultMainWithIngredients ingredients $
122+ askOption $ \(NodeOpt node) ->
123+ let (codegen, flags) = if node then (JS, ["--codegen", "node"])
124+ else (C , [])
125+ in
126+ mkGoldenTests (testFamiliesForCodegen codegen)
127+ (flags ++ idrisFlags)
128+
test/runtest.hsdeleted−230
@@ -1,230 +0,0 @@
1−{-# LANGUAGE CPP #-}
2−module Main where
3−
4−import Control.Monad
5−import Data.Char
6−import Data.List
7−import Data.Maybe
8−import qualified Data.Set as S
9−import Data.Time.Clock
10−import System.Directory
11−import System.Environment
12−import System.FilePath
13−import System.Exit
14−import System.Info
15−import System.IO
16−import System.Process
17−
18−-- Because GHC earlier than 7.8 lacks setEnv
19−-- Install the setenv package on Windows.
20−#if __GLASGOW_HASKELL__ < 708
21−#ifndef mingw32_HOST_OS
22−import qualified System.Posix.Env as PE(setEnv)
23−
24−setEnv k v = PE.setEnv k v True
25−#else
26−import System.SetEnv(setEnv)
27−#endif
28−#endif
29−
30−data Flag = Update | Diff | ShowOutput | Quiet | Time deriving (Eq, Show, Ord)
31−
32−type Flags = S.Set Flag
33−
34−data Status = Success | Failure | Updated deriving (Eq, Show)
35−
36−data Config = Config {
37− flags :: Flags,
38− idrOpts :: [String],
39− tests :: [String]
40−} deriving (Show, Eq)
41−
42−isQuiet conf = Quiet `S.member` (flags conf)
43−showOutput conf = ShowOutput `S.member` (flags conf)
44−showTime conf = Time `S.member` (flags conf)
45−showDiff conf = Diff `S.member` (flags conf)
46−doUpdate conf = Update `S.member` (flags conf)
47−
48−checkTestName :: String -> Bool
49−checkTestName d = (all isDigit $ take 3 $ reverse d)
50− && (not $ isInfixOf "disabled" d)
51−
52−enumTests :: IO [String]
53−enumTests = do
54− cwd <- getCurrentDirectory
55− dirs <- getDirectoryContents cwd
56− return $ sort $ filter checkTestName dirs
57−
58−parseFlag :: String -> Maybe Flag
59−parseFlag s = case s of
60− "-u" -> Just Update
61− "-d" -> Just Diff
62− "-s" -> Just ShowOutput
63− "-t" -> Just Time
64− "-q" -> Just Quiet
65− _ -> Nothing
66−
67−parseFlags :: [String] -> (S.Set Flag, [String])
68−parseFlags xs = (S.fromList f, i)
69− where
70− f = catMaybes $ map parseFlag fl
71− (fl, i) = partition (\s -> parseFlag s /= Nothing) xs
72−
73−parseArgs :: [String] -> IO Config
74−parseArgs args = do
75− (tests, rest) <- case args of
76− ("all":xs) -> do
77− et <- enumTests
78− return (et, xs)
79− ("without":xs) -> do
80− t <- enumTests
81− (blacklist, ys) <- return $ break (== "opts") xs
82− return (t \\ blacklist, ys \\ ["opts"])
83− (x:xs) -> do
84− exists <- doesDirectoryExist x
85− return (if checkTestName x && exists then [x] else [], xs)
86− [] -> do
87− et <- enumTests
88− return (et, [])
89− let (testOpts, idOpts) = parseFlags rest
90− return $ Config testOpts idOpts tests
91−
92−-- "bash" needed because Haskell has cmd as the default shell on windows, and
93−-- we also want to run the process with another current directory, so we get
94−-- this thing.
95−runInShell :: String -> [String] -> IO (ExitCode, String)
96−runInShell test opts = do
97− (ec, output, _) <- readCreateProcessWithExitCode
98− ((proc "bash" ("run":opts)) { cwd = Just test,
99− std_out = CreatePipe })
100− ""
101− return (ec, output)
102−
103−runTest :: Config -> String -> IO Status
104−runTest conf test = do
105− -- don't touch the current directory as we want to run these things
106− -- in parallel in the future
107− let inTest s = test ++ "/" ++ s
108− t1 <- getCurrentTime
109− (exitCode, output) <- runInShell test (idrOpts conf)
110− t2 <- getCurrentTime
111− expected <- readFile $ inTest "expected"
112− writeFile (inTest "output") output
113− res <- if (norm output == norm expected)
114− then do putStrLn $ test ++ " finished...success"
115− return Success
116− else if doUpdate conf
117− then do putStrLn $ test ++ " finished...UPDATE"
118− writeFile (inTest "expected") output
119− return Updated
120− else do putStrLn $ test ++ " finished...FAILURE"
121− _ <- rawSystem "diff" [inTest "output", inTest "expected"]
122− return Failure
123− when (showTime conf) $ do
124− let dt = diffUTCTime t2 t1
125− putStrLn $ "Duration of " ++ test ++ " was " ++ show dt
126− return res
127− where
128− -- just pretend that backslashes are slashes for comparison
129− -- purposes to avoid path problems, so don't write any tests
130− -- that depend on that distinction in other contexts.
131− -- Also rewrite newlines for consistency.
132− norm ('\r':'\n':xs) = '\n' : norm xs
133− norm ('\\':'\\':xs) = '/' : norm xs
134− norm ('\\':xs) = '/' : norm xs
135− norm (x : xs) = x : norm xs
136− norm [] = []
137−
138−printStats :: Config -> [Status] -> IO ()
139−printStats conf stats = do
140− let total = length stats
141− let successful = length $ filter (== Success) stats
142− let failures = length $ filter (== Failure) stats
143− let updates = length $ filter (== Updated) stats
144− putStrLn "\n----"
145− putStrLn $ show total ++ " tests run: " ++ show successful ++ " succesful, "
146− ++ show failures ++ " failed, " ++ show updates ++ " updated."
147− let failed = map fst $ filter ((== Failure) . snd) $ zip (tests conf) stats
148− when (failed /= []) $ do
149− putStrLn "\nFailed tests:"
150− mapM_ putStrLn failed
151− putStrLn ""
152−
153−runTests :: Config -> IO Bool
154−runTests conf = do
155− stats <- mapM (runTest conf) (tests conf)
156− unless (isQuiet conf) $ printStats conf stats
157− return $ all (== Success) stats
158−
159−runShow :: Config -> IO Bool
160−runShow conf = do
161− mapM_ (\t -> callProcess "cat" [t++"/output"]) (tests conf)
162− return True
163−
164−runDiff :: Config -> IO Bool
165−runDiff conf = do
166− mapM_ (\t -> do putStrLn $ "Differences in " ++ t ++ ":"
167− ec <- rawSystem "diff" [t++"/output", t++"/expected"]
168− when (ec == ExitSuccess) $ putStrLn "No differences found.")
169− (tests conf)
170− return True
171−
172−whisper :: Config -> String -> IO ()
173−whisper conf s = do unless (isQuiet conf) $ putStrLn s
174−
175−isWindows :: Bool
176−isWindows = os `elem` ["win32", "mingw32", "cygwin32"]
177−
178−setPath :: Config -> IO ()
179−setPath conf = do
180− maybeEnv <- lookupEnv "IDRIS"
181− idrisExists <- case maybeEnv of
182− Just idrisExe -> do
183− let exeExtension = if isWindows then ".exe" else ""
184− doesFileExist (idrisExe ++ exeExtension)
185− Nothing -> return False
186− if (idrisExists)
187− then do
188− idrisAbs <- makeAbsolute $ fromMaybe "" maybeEnv
189− setEnv "IDRIS" idrisAbs
190− whisper conf $ "Using " ++ idrisAbs
191− else do
192− path <- getEnv "PATH"
193− setEnv "IDRIS" ""
194− let sandbox = "../.cabal-sandbox/bin"
195− hasBox <- doesDirectoryExist sandbox
196− bindir <- if hasBox
197− then do
198− whisper conf $ "Using Cabal sandbox at " ++ sandbox
199− makeAbsolute sandbox
200− else do
201− stackExe <- findExecutable "stack"
202− case stackExe of
203− Just stack -> do
204− out <- readProcess stack ["path", "--dist-dir"] []
205− stackDistDir <- return $ takeWhile (/= '\n') out
206− let stackDir = "../" ++ stackDistDir ++ "/build/idris"
207− whisper conf $ "Using stack work dir at " ++ stackDir
208− makeAbsolute stackDir
209− Nothing -> return ""
210− when (bindir /= "") $ setEnv "PATH" (bindir ++ [searchPathSeparator] ++ path)
211−
212−main = do
213− hSetBuffering stdout LineBuffering
214− withCabal <- doesDirectoryExist "test"
215− when withCabal $ do
216− setCurrentDirectory "test"
217− args <- getArgs
218− conf <- parseArgs args
219− setPath conf
220− t1 <- getCurrentTime
221− res <- case tests conf of
222− [] -> return True
223− xs | showOutput conf -> runShow conf
224− xs | showDiff conf -> runDiff conf
225− xs -> runTests conf
226− t2 <- getCurrentTime
227− when (showTime conf) $ do
228− let dt = diffUTCTime t2 t1
229− putStrLn $ "Duration of Entire Test Suite was " ++ show dt
230− unless res exitFailure