on master help sign in
Merge pull request #3231 from gpyh/tasty

Ported the test infrastructure to Tasty

commit9dcbdc9c82932e891ccfe2863c0f5ebd3b31cbf1 authorJan de Muijnck-Hughes <jfdm@users.noreply.github.com> dateThu 28 Jul 2016, 18:20 (10 years ago) parent8394c9048ae753
77 files changed+799−394
.gitignore+34−21 .travis.yml+26−3 CHANGELOG.md+4 Makefile+17−5 Setup.hs+29 config.mk+4−2 idris.cabal+34−5 test/Makefile−34 test/README.md+128−1 test/TestData.hs+300 test/TestRun.hs+128 test/basic010/run+2−2 test/disambig002/disambig002.idr+2 test/docs001/run+1−1 test/docs002/run+1−1 test/docs003/run+1−1 test/docs004/run+1−1 test/docs005/run+1−1 test/effects001/run+2−2 test/effects002/run+1−1 test/ffi001/run+1−1 test/ffi002/run+1−1 test/ffi003/run+1−1 test/ffi004/run+1−1 test/ffi006/run+1−1 test/idrisdoc001/run+1−1 test/idrisdoc002/run+2−2 test/idrisdoc003/run+2−2 test/idrisdoc004/run+2−2 test/idrisdoc005/run+2−2 test/idrisdoc006/run+3−3 test/idrisdoc007/run+2−2 test/idrisdoc008/run+2−2 test/interactive001/run+1−1 test/interactive002/run+1−1 test/interactive003/run+1−1 test/interactive004/run+1−1 test/interactive005/run+1−1 test/interactive006/run+1−1 test/interactive007/run+1−1 test/interactive008/run+1−1 test/interactive009/run+1−1 test/interactive010/run+1−1 test/interactive011/run+1−1 test/interactive012/run+1−1 test/interactive013/run+1−1 test/meta002/run+1−1 test/primitives002/run+1−1 test/proof004/run+1−1 test/proof005/run+1−1 test/proof006/run+1−1 test/proof007/run+1−1 test/proof008/run+1−1 test/proofsearch003/run+1−1 test/reg002/run+1−1 test/reg003/run+2−2 test/reg007/run+1−1 test/reg023/run+1−1 test/reg034/run+1−1 test/reg035/run+3−3 test/reg039/run+2−2 test/reg040/run+1−1 test/reg044/run+1−1 test/reg049/run+1−1 test/reg050/run+3−3 test/reg054/run+1−1 test/reg055/run+2−2 test/reg075/run+1−1 test/runtest.hs−230 test/totality003/run+2−2 test/totality009/run+3−3 test/tutorial001/run+1−1 test/tutorial006/run+3−3 test/tutorial007/run+2−2 test/unique001/run+6−6 test/unique002/run+2−2 test/unique003/run+1−1
.gitignore+34−21
@@ -1,41 +1,54 @@
1−\#*
2−.\#*
1+# Build artefacts
32dist
4−cabal-dev/
5−.cabal-sandbox
6−cabal.sandbox.config
7−cabal.config
8−.hsenv
93*.ibc
104*.o
115*.a
126*.so
137*.dll
148*.dylib
15−*.swp
16−*~
17−.DS_Store
18−.hpc
19−*.orig
20−*.tix
21−*.dSYM
22−.projectile
23−.dir-locals.el
24−.vscode
25−.idea
26−custom.mk
9+src/Version_idris.hs
10+
11+# Doc generation artefacts
2712libs/base/base_doc/
2813libs/effects/effects_doc/
2914libs/prelude/prelude_doc/
3015libs/contrib/contrib_doc/
3116libs/pruviloj/pruviloj_doc/
3217libs/oldeffects/oldeffects_doc/
18+
19+# Test artefacts
3320test/output
3421test/*[0-9][0-9][0-9]/output
3522test/*[0-9][0-9][0-9]/*.exe
3623test/runtest.exe
3724test/runtest
25+.tasty-rerun-log
26+
27+# Haskell build tools
28+cabal-dev/
29+.cabal-sandbox
30+cabal.sandbox.config
31+cabal.config
32+.stack-work
33+.hsenv
34+
35+# For convenience
36+custom.mk
37+\#*
38+.\#*
3839tags
3940TAGS
40−src/Version_idris.hs
41−.stack-work
41+
42+# IDE and editors
43+*.swp
44+*~
45+.DS_Store
46+.hpc
47+*.orig
48+*.tix
49+*.dSYM
50+.projectile
51+.dir-locals.el
52+.vscode
53+.idea
54+
.travis.yml+26−3
@@ -100,15 +100,38 @@ before_script:
100100 - tar -xf Idris-dev/dist/idris*.tar.gz
101101 - cd idris*
102102script:
103− - cabal configure -f FFI -f CI
103+###
104+ - echo 'Configure...' && echo -en 'travis_fold:start:script.configure\\r'
105+ - cabal configure -f FFI -f CI --enable-tests
106+ - echo -en 'travis_fold:end:script.configure\\r'
107+###
108+ - echo 'Build...' && echo -en 'travis_fold:start:script.build\\r'
104109 - cabal build
110+ - echo -en 'travis_fold:end:script.build\\r'
111+###
112+ - echo 'Copy...' && echo -en 'travis_fold:start:script.copy\\r'
105113 - cabal copy
114+ - echo -en 'travis_fold:end:script.copy\\r'
115+###
116+ - echo 'Register...' && echo -en 'travis_fold:start:script.register\\r'
106117 - cabal register
118+ - echo -en 'travis_fold:end:script.register\\r'
119+###
120+ - echo 'Cppcheck...' && echo -en 'travis_fold:start:script.cppcheck\\r'
107121 - if [[ "$TESTS" == "test_c" ]]; then
108122 cppcheck -i 'mini-gmp.c' rts;
109123 fi
124+ - echo -en 'travis_fold:end:script.cppcheck\\r'
125+###
126+ - echo 'Tests...' && echo -en 'travis_fold:start:script.tests\\r'
110127 - for test in $TESTS; do
111− echo "make -j2 $test";
112− make -j2 $test;
128+ echo "make TEST-JOBS=2 $test";
129+ travis_wait make TEST-JOBS=2 $test;
113130 done
131+ - echo -en 'travis_fold:end:script.tests\\r'
132+###
133+ - echo 'Benchmarks...' && echo -en 'travis_fold:start:script.benchmarks\\r'
114134 - cd benchmarks && ./build.pl && ./run.pl && cd ..
135+ - echo -en 'travis_fold:end:script.benchmarks\\r'
136+###
137+#
CHANGELOG.md+4
@@ -31,6 +31,10 @@
3131* Idris' documentation system now displays the documentation for auto
3232 implicits in the output of `:doc`. This is tested for in `docs005`.
3333
34+## Miscellaneous updates
35+
36+* The test suite now uses [tasty-golden](https://hackage.haskell.org/package/tasty-golden). New tests must be registered in `test/TestData.hs`, as explained in the relevant `README.md`.
37+
3438# New in 0.12:
3539
3640## Language updates
Makefile+17−5
@@ -1,10 +1,15 @@
1−.PHONY: build configure doc install linecount nodefault pinstall lib_clean relib fast test_c test lib_doc lib_doc_clean user_doc_html user_doc_pdf user_docs
1+.PHONY: build configure doc install linecount nodefault pinstall lib_clean relib fast test_js test_c test test_clean lib_doc lib_doc_clean user_doc_html user_doc_pdf user_docs
2+
3+ARGS=
4+TEST-JOBS=
5+TEST-ARGS=
26
37include config.mk
48-include custom.mk
59
610ifdef CI
711CABALFLAGS += -f CI
12+TEST-ARGS += --color always
813endif
914
1015install:
@@ -20,13 +25,20 @@ build: dist/setup-config
2025test: doc test_c
2126
2227test_c:
23− $(MAKE) -C test IDRIS=../dist/build/idris/idris test
28+ $(CABAL) test $(ARGS) --test-options \
29+ "$(TEST-ARGS) --rerun-update +RTS -N$(TEST-JOBS) -RTS"
2430
2531test_js:
26− $(MAKE) -C test IDRIS=../dist/build/idris/idris test_js
32+ $(CABAL) test $(ARGS) --test-options \
33+ "$(TEST-ARGS) --node --rerun-update +RTS -N$(TEST-JOBS) -RTS"
34+
35+test_update:
36+ $(CABAL) test $(ARGS) --test-options \
37+ "$(TEST-ARGS) --accept +RTS -N$(TEST-JOBS) -RTS"
2738
28−test_timed:
29− $(MAKE) -C test IDRIS=../dist/build/idris/idris time
39+test_clean:
40+ rm -f test/*~
41+ rm -f test/*/output
3042
3143lib_clean:
3244 $(MAKE) -C libs IDRIS=../../dist/build/idris/idris RTS=../../dist/build/rts/libidris_rts clean
Setup.hs+29
@@ -263,6 +263,30 @@ idrisInstall verbosity copy pkg local = unless (execOnly (configFlags local)) $
263263 makeInstall src target =
264264 make verbosity [ "-C", src, "install" , "TARGET=" ++ target, "IDRIS=" ++ idrisCmd local]
265265
266+-- -----------------------------------------------------------------------------
267+-- Test
268+
269+-- FIXME: We use the __GLASGOW_HASKELL__ macro because MIN_VERSION_cabal seems
270+-- to be broken !
271+
272+-- There are two "dataDir" in cabal, and they don't relate to each other.
273+-- When fetching modules, idris uses the second path (in the pkg record),
274+-- which by default is the root folder of the project.
275+-- We want it to be the install directory where we put the idris libraries.
276+fixPkg pkg target = pkg { dataDir = target }
277+
278+-- The "Args" argument of the testHooks has been added in cabal 1.22.0,
279+-- and should therefore be ignored for prior versions.
280+#if __GLASGOW_HASKELL__ < 710
281+originalTestHook _ = testHook simpleUserHooks
282+#else
283+originalTestHook = testHook simpleUserHooks
284+#endif
285+
286+idrisTestHook args pkg local hooks flags = do
287+ let target = datadir $ L.absoluteInstallDirs pkg local NoCopyDest
288+ originalTestHook args (fixPkg pkg target) local hooks flags
289+
266290-- -----------------------------------------------------------------------------
267291-- Main
268292
@@ -281,4 +305,9 @@ main = defaultMainWithHooks $ simpleUserHooks
281305 NoCopyDest pkg local
282306 , preSDist = idrisPreSDist
283307 , postSDist = idrisPostSDist
308+#if __GLASGOW_HASKELL__ < 710
309+ , testHook = idrisTestHook ()
310+#else
311+ , testHook = idrisTestHook
312+#endif
284313 }
config.mk+4−2
@@ -14,9 +14,11 @@ CABAL :=cabal
1414# Any flags defined here which alter the RTS API must also be added to src/IRTS/CodegenC.hs
1515CFLAGS :=-O2 -Wall -DHAS_PTHREAD -DIDRIS_ENABLE_STATS $(CFLAGS)
1616
17−#CABALFLAGS :=
17+# CABALFLAGS :=
18+CABALFLAGS += --enable-tests
1819## Disable building of Effects
19−#CABALFLAGS :=-f NoEffects
20+#CABALFLAGS :=-f NoEffects $(CABALFLAGS)
21+
2022
2123ifneq (, $(findstring bsd, $(MACHINE)))
2224 GMP_INCLUDE_DIR :=
idris.cabal+34−5
@@ -146,7 +146,7 @@ Extra-source-files:
146146 libs/effects/Makefile
147147 libs/effects/effects.ipkg
148148 libs/effects/Effect/*.idr
149− libs/effects/Effect/Logging/*.idr
149+ libs/effects/Effect/Logging/*.idr
150150 libs/effects/*.idr
151151
152152 libs/pruviloj/Makefile
@@ -156,8 +156,8 @@ Extra-source-files:
156156 libs/pruviloj/Pruviloj/Derive/*.idr
157157 libs/pruviloj/Pruviloj/Internals/*.idr
158158
159− test/Makefile
160− test/runtest.hs
159+ test/TestRun.hs
160+ test/TestData.hs
161161
162162 test/regression001/run
163163 test/regression001/expected
@@ -356,6 +356,10 @@ Extra-source-files:
356356 test/bignum002/*.idr
357357 test/bignum002/expected
358358
359+ test/bounded001/run
360+ test/bounded001/*.idr
361+ test/bounded001/expected
362+
359363 test/corecords001/*.idr
360364 test/corecords001/run
361365 test/corecords001/expected
@@ -604,6 +608,9 @@ Extra-source-files:
604608 test/meta002/run
605609 test/meta002/*.idr
606610 test/meta002/expected
611+ test/meta003/run
612+ test/meta003/*.idr
613+ test/meta003/expected
607614 test/meta004/run
608615 test/meta004/*.idr
609616 test/meta004/expected
@@ -630,9 +637,21 @@ Extra-source-files:
630637 test/primitives006/expected
631638
632639 test/pkg001/run
633− test/pkg001/test-pkg.ipkg
634640 test/pkg001/*.idr
635641 test/pkg001/expected
642+ test/pkg001/*.ipkg
643+ test/pkg002/run
644+ test/pkg002/*.idr
645+ test/pkg002/expected
646+ test/pkg002/*.ipkg
647+ test/pkg003/run
648+ test/pkg003/*.idr
649+ test/pkg003/expected
650+ test/pkg003/*.ipkg
651+ test/pkg004/run
652+ test/pkg004/*.idr
653+ test/pkg004/expected
654+ test/pkg004/*.ipkg
636655
637656 test/proof001/run
638657 test/proof001/*.idr
@@ -1139,7 +1158,8 @@ Executable idris
11391158
11401159Test-suite regression-and-sanity-tests
11411160 Type: exitcode-stdio-1.0
1142− Main-is: runtest.hs
1161+ Main-is: TestRun.hs
1162+ Other-modules: TestData
11431163 hs-source-dirs: test
11441164
11451165 Build-depends: idris
@@ -1150,7 +1170,16 @@ Test-suite regression-and-sanity-tests
11501170 , filepath
11511171 , directory
11521172 , haskeline >= 0.7
1173+ , optparse-applicative >= 0.11 && < 0.13
1174+ , tagged
1175+ , tasty >= 0.8
1176+ , tasty-golden >= 2.0
1177+ , tasty-rerun >= 1.0.0
1178+ , bytestring
11531179 , transformers
1180+ if impl(ghc < 7.10)
1181+ Extensions: DeriveDataTypeable
1182+
11541183
11551184 ghc-prof-options: -auto-all -caf-all
11561185 ghc-options: -threaded -rtsopts -funbox-strict-fields
test/Makefiledeleted−34
@@ -1,34 +0,0 @@
1−.PHONY: test test_js time update diff distclean $(TESTS)
2−
3−TESTS = $(sort $(patsubst %/,%.test,$(wildcard */)))
4−
5−test: $(TESTS)
6−
7−
8−info: runtest
9− @./runtest all
10−
11−%.test: runtest
12− @./runtest $(patsubst %.test,%,$@) -q
13−
14−test_js: runtest
15− @./runtest without tutorial007 sugar004 reg029 reg052 io001 dsl002 io003 effects001 effects002 basic007 basic011 ffi006 ffi007 ffi008 primitives005 primitives006 views003 opts --codegen node
16−
17−update: runtest
18− @./runtest all -u
19−
20−diff: runtest
21− @./runtest all -d
22−
23−time: runtest
24− @./runtest all -t
25−
26−distclean:
27− @rm runtest
28− @rm -f *~
29− @rm -f */output
30−
31−
32−runtest:
33− @ghc --make runtest.hs
34− @rm runtest.o runtest.hi
test/README.md+128−1
@@ -1,6 +1,66 @@
11# Idris' Testing Suite
22
3−Tests are categorised as follows:
3+## Running the test suite
4+
5+Do `make test_c` or `make test_js`. It will run the tests in parallel and through `cabal test`.
6+You can also manually call `cabal test` or `stack test`.
7+
8+### Passing arguments
9+
10+#### To the test program
11+
12+You can pass arguments to the test program. For example, to pass the `--help` argument:
13+
14+```
15+# Via make
16+make TEST-ARGS="--help" test_c
17+# Via cabal
18+cabal test --test-options="--help"
19+# Via stack
20+stack test --test-arguments="--help"
21+```
22+
23+Try it to learn more about the other arguments you can provide. Two are of particular interest: `--node` to test against the Node code generator and `--pattern <regex>` to only run the test that match the provided `<regex>`.
24+
25+#### To idris
26+
27+You can pass arguments to idris in each of its invocation by the tests. There are two ways to this. You can either modify the `idrisFlags` term in `TestRun`, or set the `$IDRIS` environment variable accordingly
28+
29+### Specifying the number of parallel jobs
30+
31+With make, the test suite runs in parallel by default. You can specify the number of threads with `TEST-JOBS`. For stack and cabal, you need to explicitly enable parallelism as you would do with any other GHC-compiled executable. Examples:
32+
33+```
34+# Two test jobs via make
35+make TEST-JOBS=2 test_c
36+# Two test jobs via cabal
37+cabal test --test-options="+RTS -N2 -RTS"
38+# Two test jobs via stack
39+stack test --test-arguments="+RTS -N2 -RTS"
40+```
41+
42+### Running only previously failed tests
43+
44+Because of the `--rerun-update` option, `make test_c` will create a `.tasty-rerun-log` file in the root directory of the project. Each time the test suite is run, the file will be written with the result of the tests. The next time you do `make test`, you can specify the `rerun-filter` argument to, for example, only run previously failed tests. Valid values are given in the `--help`.
45+
46+## Extending the test suite
47+
48+### Content of the directory
49+
50+To each test there shall be a folder that holds at least a bash script `run` and a golden file `expected`. When running the test suite, the standard output of the script is compared against the golden file. Any mismatch is reported as a failure.
51+
52+The name of a test folder is the identifier of the *test family* followed by a three digit number. A test family gathers related tests.
53+
54+Add to that four top level files:
55+
56+- The program `TestRun.hs` that runs the test suite
57+- A Haskell module `TestData.hs` with the metadata for each test
58+- The Perl script `mktest.pl` to ease the creation of a new test
59+- This `README.md`
60+
61+### Test families
62+
63+Tests are categorised with the following test families (and their identifiers):
464
565+ *basic*: Basic language features, some complete programs
666+ *bignum*: Bignums and GMP
@@ -8,6 +68,7 @@ Tests are categorised as follows:
868+ *corecords*: Tests for corecord
969+ *delab*: De-elaboration tests
1070+ *directives*: Tests for directives
71++ *disambig*: Disambiguaton tests
1172+ *docs*: Testing documentation functionality
1273+ *dsl*: Embedded DSLs and features to support DSL development
1374+ *effects*: Effects package
@@ -31,9 +92,75 @@ Tests are categorised as follows:
3192+ *regression*: Regression tests, covering previous bug fixes
3293+ *sourceLocation*: Interaction with files from Idris
3394+ *sugar*: Syntactic sugar, syntax extensions
95++ *syntax*: Syntax extensions
3496+ *tactics*: Testing for tactics
3597+ *totality*: Totality checking
3698+ *tutorial*: Examples from the tutorial
3799+ *unique*: Uniqueness types
38100+ *universes*: Universes
39101+ *views*: Views and covering functions
102+
103+### Adding a test family
104+
105+1. Choose an available identifier for your test family. It should be short
106+and somewhat self-explanatory.
107+2. Add it in the `README.md` file, i.e. just above.
108+3. Add it in `TestData.hs`, in the `testFamiliesData` list. Say the chosen identifier is `foo`. You should append to the list `("foo", "A proper name for foo", [ ])`. The empty list will be replaced with the metadata of each test in the family.
109+4. Add your tests.
110+
111+### Adding a test
112+
113+1. Choose the family your test shall belong to and remember its identifier.
114+2. Pick the next available integer in the test family. It will be the index.
115+3. Say the family's identifier is `foo` and the index is `42`. You should call `./mktest.pl foo042` ; this will create the directory and a simple `run` script.
116+4. Modify the `run` script to your liking. If you want to call the idris executable, write `${IDRIS:-idris} $@`.
117+5. Add any file you may need in the directory. If they don't end in `.idr`, you should remember them for the next step.
118+6. Add your test in `TestData.hs`. Each family has a list of `(Index, CompatCodegen)`. See the next section for the available values in `CompatCodegen`. In most cases, you should write `( 42, ANY)`.
119+7. Generate the `expected` file by doing:
120+ ```
121+ # Using cabal
122+ cabal test --test-options="--pattern foo042 --accept"
123+ # Using stack
124+ stack test --test-arguments"--pattern foo042 --accept"
125+ ```
126+8. Check the content of `expected`. Maybe the test didn't do what you thought it would. Fix and go back to 7 until it's ok.
127+9. Add under `Extra-source-files` in `idris.cabal` the patterns that match the folder's content. If you forget this, your test will fail in Travis CI. With the previous example, it should be at least:
128+ ```
129+ test/foo042/run
130+ test/foo042/expected
131+ test/foo042/*.idr
132+ ```
133+
134+### Specifying compatible backends
135+
136+Some tests only make sense with specific code generators. Others don't generate code. You need to supply this information in `testFamiliesData`. Available values
137+for the compatible backends are:
138+
139+- `ANY`: choose this if your test will work with any code generator
140+- `C_CG`: choose this to only test against the C code generator
141+- `JS_CG`: choose this to only test against the Node code generator
142+- `NONE`: choose this if you don't want your test to be run with multiple code
143+generators (mainly for tests that only perform type checking)
144+
145+Currently, `NONE` has the same effect as `ANY`, but this will change.
146+
147+### Removing a test
148+
149+1. Delete the corresponding directory
150+2. Remove the corresponding line in the definition of `testFamiliesData` in `TestData.hs`
151+3. Remove the related lines under `Extra-source-files` in `idris.cabal`
152+
153+### Updating golden files
154+
155+To update the `expected` file for every test, do one of the following:
156+
157+```
158+# Using make
159+make test_update
160+# Using cabal
161+cabal test --test-options="--accept"
162+# Using stack
163+stack test --test-arguments="--accept"
164+```
165+
166+"Accepted" tests are the ones that update the golden file. A test can still fail if the `run` script itself crashes.
test/TestData.hsnew file+300
@@ -0,0 +1,300 @@
1+module TestData where
2+
3+import Data.IntMap as IMap
4+import Data.Map.Strict as Map
5+import Data.Set as Set
6+
7+data Codegen = C | JS deriving (Show, Eq, Ord)
8+type Index = Int
9+data CompatCodegen = ANY | C_CG | NODE_CG | NONE
10+
11+-- A TestFamily groups tests that share the same theme
12+data TestFamily = TestFamily {
13+ -- A shorter lowcase name to use in filenames
14+ id :: String,
15+ -- A proper name for the test family that will be displayed
16+ name :: String,
17+ -- A map of test metadata:
18+ -- - The key is the index (>=1 && <1000)
19+ -- - The value is the set of compatible code generators,
20+ -- or Nothing if the test doesn't depend on a code generator
21+ tests :: IntMap (Maybe (Set Codegen))
22+} deriving (Show)
23+
24+toCodegenSet :: CompatCodegen -> Maybe (Set Codegen)
25+toCodegenSet compatCodegen = fmap Set.fromList mList where
26+ mList = case compatCodegen of
27+ ANY -> Just [ C, JS ]
28+ C_CG -> Just [ C ]
29+ NODE_CG -> Just [ JS ]
30+ NONE -> Nothing
31+
32+testFamilies :: [TestFamily]
33+testFamilies = fmap instanciate testFamiliesData where
34+ instanciate (id, name, testsData) = TestFamily id name tests where
35+ tests = IMap.fromList (fmap makeSetCodegen testsData)
36+ makeSetCodegen (index, codegens) = (index, toCodegenSet codegens)
37+
38+testFamiliesForCodegen :: Codegen -> [TestFamily]
39+testFamiliesForCodegen codegen =
40+ fmap (\testFamily -> testFamily {tests = IMap.filter f (tests testFamily)})
41+ testFamilies
42+ where
43+ f mCodegens = case mCodegens of
44+ Just codegens -> Set.member codegen codegens
45+ Nothing -> True
46+
47+-- The data to instanciate testFamilies
48+-- The first column is the id
49+-- The second column is the proper name (the prefix of the subfolders)
50+-- The third column is the data for each test
51+testFamiliesData :: [(String, String, [(Index, CompatCodegen)])]
52+testFamiliesData = [
53+ ("basic", "Basic",
54+ [ ( 1, ANY ),
55+ ( 2, ANY ),
56+ ( 3, ANY ),
57+ ( 4, ANY ),
58+ ( 5, ANY ),
59+ ( 6, ANY ),
60+ ( 7, C_CG ),
61+ ( 8, ANY ),
62+ ( 9, ANY ),
63+ ( 10, ANY ),
64+ ( 11, C_CG ),
65+ ( 12, ANY ),
66+ ( 13, ANY ),
67+ ( 14, ANY ),
68+ ( 15, ANY ),
69+ ( 16, ANY ),
70+ ( 17, ANY ),
71+ ( 18, ANY )]),
72+ ("bignum", "Bignum",
73+ [ ( 1, ANY ),
74+ ( 2, ANY )]),
75+ ("bounded", "Bounded",
76+ [ ( 1, ANY )]),
77+ ("corecords", "Corecords",
78+ [ ( 1, ANY ),
79+ ( 2, ANY )]),
80+ ("delab", "De-elaboration",
81+ [ ( 1, ANY )]),
82+ ("directives", "Directives",
83+ [ ( 1, ANY ),
84+ ( 2, ANY )]),
85+ ("disambig", "Disambiguation",
86+ [ ( 2, ANY )]),
87+ ("docs", "Documentation",
88+ [ ( 1, ANY ),
89+ ( 2, ANY ),
90+ ( 3, ANY ),
91+ ( 4, ANY ),
92+ ( 5, ANY )]),
93+ ("dsl", "DSL",
94+ [ ( 1, ANY ),
95+ ( 2, C_CG ),
96+ ( 3, ANY ),
97+ ( 4, ANY )]),
98+ ("effects", "Effects",
99+ [ ( 1, C_CG ),
100+ ( 2, C_CG ),
101+ ( 3, ANY ),
102+ ( 4, ANY ),
103+ ( 5, ANY )]),
104+ ("error", "Errors",
105+ [ ( 1, ANY ),
106+ ( 2, ANY ),
107+ ( 3, ANY ),
108+ ( 4, ANY ),
109+ ( 5, ANY ),
110+ ( 6, ANY ),
111+ ( 7, ANY ),
112+ ( 8, ANY )]),
113+ ("ffi", "FFI",
114+ [ ( 1, ANY ),
115+ ( 2, ANY ),
116+ ( 3, ANY ),
117+ ( 4, ANY ),
118+ ( 5, ANY ),
119+ ( 6, C_CG ),
120+ ( 7, C_CG ),
121+ ( 8, C_CG )]),
122+ ("folding", "Folding",
123+ [ ( 1, ANY )]),
124+ ("idrisdoc", "Idris documentation",
125+ [ ( 1, ANY ),
126+ ( 2, ANY ),
127+ ( 3, ANY ),
128+ ( 4, ANY ),
129+ ( 5, ANY ),
130+ ( 6, ANY ),
131+ ( 7, ANY ),
132+ ( 8, ANY ),
133+ ( 9, ANY )]),
134+ ("interactive", "Interactive editing",
135+ [ ( 1, ANY ),
136+ ( 2, ANY ),
137+ ( 3, ANY ),
138+ ( 4, ANY ),
139+ ( 5, ANY ),
140+ ( 6, ANY ),
141+ ( 7, ANY ),
142+ ( 8, ANY ),
143+ ( 9, ANY ),
144+ ( 10, ANY ),
145+ ( 11, ANY ),
146+ ( 12, ANY ),
147+ ( 13, ANY )]),
148+ ("interfaces", "Interfaces",
149+ [ ( 1, ANY ),
150+ ( 2, ANY ),
151+ ( 3, ANY ),
152+ ( 4, ANY ),
153+ ( 5, ANY ),
154+ ( 6, ANY )]),
155+ ("io", "IO monad",
156+ [ ( 1, C_CG ),
157+ ( 2, ANY ),
158+ ( 3, C_CG )]),
159+ ("literate", "Literate programming",
160+ [ ( 1, ANY )]),
161+ ("meta", "Meta-programming",
162+ [ ( 1, ANY ),
163+ ( 2, ANY ),
164+ ( 3, ANY ),
165+ ( 4, ANY )]),
166+ ("pkg", "Packages",
167+ [ ( 1, ANY ),
168+ ( 2, ANY ),
169+ ( 3, ANY ),
170+ ( 4, ANY )]),
171+ ("primitives", "Primitive types",
172+ [ ( 1, ANY ),
173+ ( 2, ANY ),
174+ ( 3, ANY ),
175+ ( 5, C_CG ),
176+ ( 6, C_CG )]),
177+ ("proof", "Theorem proving",
178+ [ ( 1, ANY ),
179+ ( 2, ANY ),
180+ ( 3, ANY ),
181+ ( 4, ANY ),
182+ ( 5, ANY ),
183+ ( 6, ANY ),
184+ ( 7, ANY ),
185+ ( 8, ANY ),
186+ ( 9, ANY ),
187+ ( 10, ANY ),
188+ ( 11, ANY )]),
189+ ("proofsearch", "Proof search",
190+ [ ( 1, ANY ),
191+ ( 2, ANY ),
192+ ( 3, ANY )]),
193+ ("pruviloj", "Pruviloj",
194+ [ ( 1, ANY )]),
195+ ("quasiquote", "Quasiquotations",
196+ [ ( 1, ANY ),
197+ ( 2, ANY ),
198+ ( 3, ANY ),
199+ ( 4, ANY ),
200+ ( 5, ANY ),
201+ ( 6, ANY )]),
202+ ("records", "Records",
203+ [ ( 1, ANY ),
204+ ( 2, ANY ),
205+ ( 3, ANY ),
206+ ( 4, ANY ),
207+ ( 5, ANY )]),
208+ ("reg", "Regressions",
209+ [ ( 2, ANY ),
210+ ( 3, ANY ),
211+ ( 4, ANY ),
212+ ( 5, ANY ),
213+ ( 6, ANY ),
214+ ( 7, ANY ),
215+ ( 10, ANY ),
216+ ( 13, ANY ),
217+ ( 16, ANY ),
218+ ( 17, ANY ),
219+ ( 18, ANY ),
220+ ( 20, ANY ),
221+ ( 23, ANY ),
222+ ( 24, ANY ),
223+ ( 25, ANY ),
224+ ( 27, ANY ),
225+ ( 28, ANY ),
226+ ( 29, C_CG ),
227+ ( 31, ANY ),
228+ ( 32, ANY ),
229+ ( 34, ANY ),
230+ ( 35, ANY ),
231+ ( 39, ANY ),
232+ ( 40, ANY ),
233+ ( 41, ANY ),
234+ ( 42, ANY ),
235+ ( 44, ANY ),
236+ ( 45, ANY ),
237+ ( 48, ANY ),
238+ ( 49, ANY ),
239+ ( 50, ANY ),
240+ ( 52, C_CG ),
241+ ( 54, ANY ),
242+ ( 55, ANY ),
243+ ( 56, ANY ),
244+ ( 67, ANY ),
245+ ( 68, ANY ),
246+ ( 69, ANY ),
247+ ( 70, ANY ),
248+ ( 72, ANY ),
249+ ( 75, ANY )]),
250+ ("regression", "Regression (loner)",
251+ [ ( 1 , ANY )]),
252+ ("sourceLocation", "Source location",
253+ [ ( 1 , ANY )]),
254+ ("sugar", "Syntactic sugar",
255+ [ ( 1, ANY ),
256+ ( 2, ANY ),
257+ ( 3, ANY ),
258+ ( 4, C_CG ),
259+ ( 5, ANY )]),
260+ ("syntax", "Syntax extensions",
261+ [ ( 1, ANY ),
262+ ( 2, ANY )]),
263+ ("tactics", "Tactics",
264+ [ ( 1, ANY )]),
265+ ("totality", "Totality checking",
266+ [ ( 1, ANY ),
267+ ( 2, ANY ),
268+ ( 3, ANY ),
269+ ( 4, ANY ),
270+ ( 5, ANY ),
271+ ( 6, ANY ),
272+ ( 7, ANY ),
273+ ( 8, ANY ),
274+ ( 9, ANY ),
275+ ( 10, ANY ),
276+ ( 11, ANY ),
277+ ( 12, ANY ),
278+ ( 13, ANY ),
279+ ( 14, ANY ),
280+ ( 15, ANY )]),
281+ ("tutorial", "Tutorial examples",
282+ [ ( 1, ANY ),
283+ ( 2, ANY ),
284+ ( 3, ANY ),
285+ ( 4, ANY ),
286+ ( 5, ANY ),
287+ ( 6, ANY ),
288+ ( 7, C_CG )]),
289+ ("unique", "Uniqueness types",
290+ [ ( 1, ANY ),
291+ ( 2, ANY ),
292+ ( 3, ANY )]),
293+ ("universes", "Universes",
294+ [ ( 1, ANY ),
295+ ( 2, ANY )]),
296+ ("views", "Views",
297+ [ ( 1, ANY ),
298+ ( 2, ANY ),
299+ ( 3, C_CG )])]
300+
test/TestRun.hsnew file+128
@@ -0,0 +1,128 @@
1+module Main where
2+
3+import Control.Monad
4+import Data.Typeable
5+import Data.Proxy
6+import Data.List
7+import Data.Map.Strict as Map
8+import Data.IntSet as ISet
9+import Data.IntMap as IMap
10+import qualified Data.ByteString as BS
11+import qualified Data.ByteString.Char8 as BSC
12+
13+import System.Directory
14+import System.Environment
15+import System.Process
16+import System.Info
17+import System.IO
18+import System.FilePath ((</>))
19+import Options.Applicative
20+import Test.Tasty
21+import Test.Tasty.Golden
22+import Test.Tasty.Golden.Advanced
23+import Test.Tasty.Runners
24+import Test.Tasty.Options
25+import Test.Tasty.Ingredients.Rerun
26+
27+import TestData
28+
29+--------------------------------------------------------------------- [ Config ]
30+
31+type Flags = [String]
32+
33+-- Add arguments to calls of idris executable
34+idrisFlags :: Flags
35+idrisFlags = []
36+
37+testDirectory :: String
38+testDirectory = "test"
39+
40+-------------------------------------------------------------------- [ Options ]
41+
42+-- The `--node` option makes idris use the node code generator
43+-- As a consequence, incompatible tests are removed
44+
45+newtype NodeOpt = NodeOpt Bool deriving (Eq, Ord, Typeable)
46+
47+nodeArg = "node"
48+nodeHelp = "Performs the tests with the node code generator"
49+instance IsOption NodeOpt where
50+ defaultValue = NodeOpt False
51+ parseValue = fmap NodeOpt . safeRead
52+ optionName = return nodeArg
53+ optionHelp = return nodeHelp
54+ optionCLParser = fmap NodeOpt $ switch (long nodeArg <> help nodeHelp)
55+
56+ingredients :: [Ingredient]
57+ingredients = [rerunningTests [consoleTestReporter],
58+ includingOptions [Option (Proxy :: Proxy NodeOpt)] ]
59+
60+----------------------------------------------------------------------- [ Core ]
61+
62+-- Compare a given file contents against the golden file contents
63+-- A ripoff of goldenVsFile from Tasty.Golden
64+test :: String -> String -> IO () -> TestTree
65+test name path act =
66+ goldenTest name (BS.readFile ref) (act >> BS.readFile new) cmp upd
67+ where
68+ ref = path </> "expected"
69+ new = path </> "output"
70+ cmp x y = return $ if x == y then Nothing
71+ else Just $ printDiff (ref, x) (new, y)
72+ upd = BS.writeFile ref
73+
74+-- Takes the filepath and content of `expected` and `output`
75+-- and formats an error message stating their difference
76+printDiff :: (String, BS.ByteString) -> (String, BS.ByteString) -> String
77+printDiff (ref, x) (new, y) =
78+ let refcnt = BSC.unpack x
79+ newcnt = BSC.unpack y
80+ printContent cnt =
81+ if Data.List.null cnt
82+ then " is empty...\n"
83+ else " is: \n" ++ unlines (fmap ((++) " ") (lines cnt))
84+ in
85+ "Test mismatch!\n" ++
86+ "Golden file " ++ ref ++ printContent refcnt ++
87+ "However, " ++ new ++ printContent newcnt
88+
89+-- Should always output a 3-charater string from a postive Int
90+indexToString :: Int -> String
91+indexToString index = let str = show index in
92+ (replicate (3 - length str) '0') ++ str
93+
94+-- Turns the collection of TestFamily into actual tests usable by Tasty
95+mkGoldenTests :: [TestFamily] -> Flags -> TestTree
96+mkGoldenTests testFamilies flags =
97+ testGroup "Regression and sanity tests"
98+ (fmap mkTestFamily testFamilies)
99+ where
100+ mkTestFamily (TestFamily id name tests) =
101+ testGroup name (fmap (mkTest id) (IMap.keys tests))
102+ mkTest id index =
103+ let testname = id ++ indexToString index
104+ path = testDirectory </> testname
105+ in
106+ test testname path (runTest path flags)
107+
108+-- Runs a test script
109+-- "bash" needed because Haskell has cmd as the default shell on windows, and
110+-- we also want to run the process with another current directory, so we get
111+-- this thing.
112+runTest :: String -> Flags -> IO ()
113+runTest path flags = do
114+ let run = (proc "bash" ("run" : flags)) {cwd = Just path,
115+ std_out = CreatePipe}
116+ (_, output, _) <- readCreateProcessWithExitCode run ""
117+ writeFile (path </> "output") output
118+
119+main :: IO ()
120+main = do
121+ defaultMainWithIngredients ingredients $
122+ askOption $ \(NodeOpt node) ->
123+ let (codegen, flags) = if node then (JS, ["--codegen", "node"])
124+ else (C , [])
125+ in
126+ mkGoldenTests (testFamiliesForCodegen codegen)
127+ (flags ++ idrisFlags)
128+
test/basic010/run+2−2
@@ -8,7 +8,7 @@ do
88done
99
1010if (which timeout&>/dev/null); then
11− timeout 10 idris $@ Main.idr --nocolour --check --warnreach -o basic010
11+ timeout 10 ${IDRIS:-idris} $@ Main.idr --nocolour --check --warnreach -o basic010
1212 timeout 5 ./basic010
1313else
1414 # From http://unix.stackexchange.com/questions/43340/how-to-introduce-timeout-for-shell-scripting
@@ -35,7 +35,7 @@ else
3535
3636 }
3737
38− timeout 10 "idris ${extraargs[*]} Main.idr --nocolour --check --warnreach -o basic010"
38+ timeout 10 "${IDRIS:-idris} ${extraargs[*]} Main.idr --nocolour --check --warnreach -o basic010"
3939 timeout 5 "./basic010"
4040fi
4141rm -f *.ibc basic010
test/disambig002/disambig002.idr+2
@@ -2,6 +2,8 @@ import Data.Vect
22import Data.Fin
33import Control.Isomorphism
44
5+-- Tests if Idris can disambiguate between the three `toVect` operators
6+
57Dec0 : Type -> Type
68Dec0 = Dec
79
test/docs001/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --nocolor docs001.idr < input
2+${IDRIS:-idris} $@ --quiet --nocolor docs001.idr < input
33rm *.ibc
test/docs002/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nobanner --nocolor --quiet docs002.idr < input
2+${IDRIS:-idris} $@ --nobanner --nocolor --quiet docs002.idr < input
33rm *.ibc
test/docs003/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --nocolor docs003.idr < input
2+${IDRIS:-idris} $@ --quiet --nocolor docs003.idr < input
33rm *.ibc
test/docs004/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --nocolor docs004.idr < input
2+${IDRIS:-idris} $@ --quiet --nocolor docs004.idr < input
33rm *.ibc
test/docs005/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --nocolor docs005.idr < input
2+${IDRIS:-idris} $@ --quiet --nocolor docs005.idr < input
33rm *.ibc
test/effects001/run+2−2
@@ -1,6 +1,6 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} -p effects $@ test021.idr -o test021
3−${IDRIS:-idris} -p effects $@ test021a.idr -o test021a
2+${IDRIS:-idris} $@ -p effects test021.idr -o test021
3+${IDRIS:-idris} $@ -p effects test021a.idr -o test021a
44./test021
55./test021a
66rm -f test021 test021a *.ibc
test/effects002/run+1−1
@@ -1,4 +1,4 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} -p effects $@ test025.idr -o test025
2+${IDRIS:-idris} $@ -p effects test025.idr -o test025
33./test025
44rm -f test025 *.ibc
test/ffi001/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --nocolour test022.idr --exec main
2+${IDRIS:-idris} $@ --quiet --nocolour test022.idr --exec main
33rm -f *.ibc
test/ffi002/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet test023.idr -o test023
2+${IDRIS:-idris} $@ --quiet test023.idr -o test023
33rm -f test023 *.ibc
test/ffi003/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --nocolour test024.idr --exec main < input
2+${IDRIS:-idris} $@ --quiet --nocolour test024.idr --exec main < input
33rm -f *.ibc
test/ffi004/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --nocolour --check test026.idr
2+${IDRIS:-idris} $@ --quiet --nocolour --check test026.idr
33rm -f *.ibc
test/ffi006/run+1−1
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
22${IDRIS:-idris} $@ ffi006.idr --interface -o ffi006.o
3−${CC:=cc} ffi006.c ffi006.o `idris --include` `idris --link` -o ffi006
3+${CC:=cc} ffi006.c ffi006.o `${IDRIS:-idris} $@ --include` `${IDRIS:-idris} $@ --link` -o ffi006
44./ffi006
55rm -f ffi006 *.ibc *.o *.h
test/idrisdoc001/run+1−1
@@ -1,4 +1,4 @@
11#!/usr/bin/env bash
22# Tests that no documentation is built for empty and/or private-only namespaces
3−${IDRIS:-idris} --mkdoc test_empty.ipkg
3+${IDRIS:-idris} $@ --mkdoc test_empty.ipkg
44rm -rf *.ibc *_doc
test/idrisdoc002/run+2−2
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
22# Tests that documentation is generated for functions
3−${IDRIS:-idris} --mkdoc test_functions.ipkg
3+${IDRIS:-idris} $@ --mkdoc test_functions.ipkg
44[ -d test_functions_doc ] && echo "Functions are documented"
5−rm -rf *.ibc *_doc
5+rm -rf *.ibc *_doc
test/idrisdoc003/run+2−2
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
22# Tests that documentation is generated for data types
3−${IDRIS:-idris} --mkdoc test_datatypes.ipkg
3+${IDRIS:-idris} $@ --mkdoc test_datatypes.ipkg
44[ -d test_datatypes_doc ] && echo "Data types are documented"
5−rm -rf *.ibc *_doc
5+rm -rf *.ibc *_doc
test/idrisdoc004/run+2−2
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
22# Tests that documentation is generated for typeclasses
3−${IDRIS:-idris} --mkdoc test_typeclasses.ipkg
3+${IDRIS:-idris} $@ --mkdoc test_typeclasses.ipkg
44[ -d test_typeclasses_doc ] && echo "Typeclasses are documented"
5−rm -rf *.ibc *_doc
5+rm -rf *.ibc *_doc
test/idrisdoc005/run+2−2
@@ -1,6 +1,6 @@
11#!/usr/bin/env bash
22# Tests that references to other namespaces are traced
3−${IDRIS:-idris} --mkdoc test_tracing.ipkg
3+${IDRIS:-idris} $@ --mkdoc test_tracing.ipkg
44[ -f test_tracing_doc/docs/TestTracing.html ] && echo "TestTracing: Check"
55[ -f test_tracing_doc/docs/Prelude.Bool.html ] && echo "Prelude.Bool: Check"
6−rm -rf *.ibc *_doc
6+rm -rf *.ibc *_doc
test/idrisdoc006/run+3−3
@@ -1,8 +1,8 @@
11#!/usr/bin/env bash
22# Tests that documentation properly is merged with existent.
3−${IDRIS:-idris} --mkdoc package_a.ipkg
3+${IDRIS:-idris} $@ --mkdoc package_a.ipkg
44[ -f test_merge_doc/IdrisDoc ] && echo "IdrisDoc file written"
5−${IDRIS:-idris} --mkdoc package_b.ipkg
5+${IDRIS:-idris} $@ --mkdoc package_b.ipkg
66ls -1p test_merge_doc/docs
77if grep -q "href\\=\"docs/A.fully.Qualified.NAME\\.html\"" test_merge_doc/index.html; then
88 echo A.fully.Qualified.NAME is in the index
@@ -14,4 +14,4 @@ if grep -q "href\\=\"docs/B\\.html\"" test_merge_doc/index.html; then
1414else
1515 echo B is NOT in the index
1616fi
17−rm -rf *.ibc *_doc A/fully/Qualified/NAME.ibc
17+rm -rf *.ibc *_doc A/fully/Qualified/NAME.ibc
test/idrisdoc007/run+2−2
@@ -1,6 +1,6 @@
11#!/usr/bin/env bash
22# Tests that documentation only is written when safe
33mkdir do_not_delete_doc
4−${IDRIS:-idris} --mkdoc package.ipkg > nowhere
4+${IDRIS:-idris} $@ --mkdoc package.ipkg > nowhere
55echo Exit status \(expects 1\): $?
6−rm -rf do_not_delete_doc *.ibc nowhere
6+rm -rf do_not_delete_doc *.ibc nowhere
test/idrisdoc008/run+2−2
@@ -1,6 +1,6 @@
11#!/usr/bin/env bash
22# Tests that documentation is generated for both public and abstract members.
3−${IDRIS:-idris} --mkdoc visibility.ipkg
3+${IDRIS:-idris} $@ --mkdoc visibility.ipkg
44[ -f visibility_doc/docs/Abstract.html ] && echo "Abstract members are documented"
55[ -f visibility_doc/docs/Visible.html ] && echo "Public members are documented"
6−rm -rf *.ibc *_doc
6+rm -rf *.ibc *_doc
test/interactive001/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet test032.idr < input
2+${IDRIS:-idris} $@ --quiet test032.idr < input
33rm -f *.ibc
test/interactive002/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet interactive002.idr < input
2+${IDRIS:-idris} $@ --quiet interactive002.idr < input
33rm -f *.ibc
test/interactive003/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet interactive003.idr < input
2+${IDRIS:-idris} $@ --quiet interactive003.idr < input
33rm -f *.ibc
test/interactive004/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet interactive004.idr < input
2+${IDRIS:-idris} $@ --quiet interactive004.idr < input
33rm -f *.ibc
test/interactive005/run+1−1
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --quiet interactive005.idr --consolewidth 70 < input
2+${IDRIS:-idris} $@ --nocolour --quiet interactive005.idr --consolewidth 70 < input
33rm -f *.ibc
44rm -f hello.bytecode
55rm -f hello
test/interactive006/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --port 5000 interactive006.idr < input
2+${IDRIS:-idris} $@ --quiet --port 5000 interactive006.idr < input
33rm -f *.ibc
test/interactive007/run+1−1
@@ -1,2 +1,2 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} -p contrib --nobanner --nocolor < input
2+${IDRIS:-idris} $@ -p contrib --nobanner --nocolor < input
test/interactive008/run+1−1
@@ -1,2 +1,2 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nobanner --nocolor < input
2+${IDRIS:-idris} $@ --nobanner --nocolor < input
test/interactive009/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet interactive009.idr < input
2+${IDRIS:-idris} $@ --quiet interactive009.idr < input
33rm -f *.ibc
test/interactive010/run+1−1
@@ -1,2 +1,2 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet --nobanner --nocolor < input
2+${IDRIS:-idris} $@ --quiet --nobanner --nocolor < input
test/interactive011/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet interactive011.idr < input
2+${IDRIS:-idris} $@ --quiet interactive011.idr < input
33rm -f *.ibc
test/interactive012/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --consolewidth 70 --quiet interactive012.idr < input
2+${IDRIS:-idris} $@ --nocolour --consolewidth 70 --quiet interactive012.idr < input
33rm -f *.ibc
test/interactive013/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet interactive013.idr < input
2+${IDRIS:-idris} $@ --quiet interactive013.idr < input
33rm -f *.ibc
test/meta002/run+1−1
@@ -3,5 +3,5 @@ ${IDRIS:-idris} $@ -p pruviloj --nocolour --check --consolewidth 70 DataDef.idr
33${IDRIS:-idris} $@ -p pruviloj --nocolour --check --consolewidth 70 Tacs.idr
44${IDRIS:-idris} $@ -p pruviloj --nocolour --check --consolewidth 70 AgdaStyleReflection.idr
55# Disabled due to excess memory consumption
6−# idris $@ -p pruviloj --nocolour --check --consolewidth 70 Deriving.idr
6+# ${IDRIS:-idris} $@ -p pruviloj --nocolour --check --consolewidth 70 Deriving.idr
77rm -f *.ibc
test/primitives002/run+1−1
@@ -48,7 +48,7 @@ for T in "${TESTS[@]}"
4848do
4949 echo ${T%% *} ${T##*#}
5050 generate_testfile "tmptest.idr" "${T%%#*}"
51− idris "$@" --quiet tmptest.idr -o tmptest || echo "missing primitive in ${CG}"
51+ ${IDRIS:-idris} $@ --quiet tmptest.idr -o tmptest || echo "missing primitive in ${CG}"
5252 ./tmptest <<<"${T##*#}"
5353 rm tmptest.idr tmptest.ibc tmptest
5454done
test/proof004/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --check $@ test035.idr
2+${IDRIS:-idris} $@ --check test035.idr
33rm -f *.ibc
test/proof005/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --check $@ DefaultArgSubstitutionSuccess.idr
2+${IDRIS:-idris} $@ --check DefaultArgSubstitutionSuccess.idr
33rm -f *.ibc
test/proof006/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --check $@ DefaultArgSubstitutionSyntax.idr
2+${IDRIS:-idris} $@ --check DefaultArgSubstitutionSyntax.idr
33rm -f *.ibc
test/proof007/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --check $@ DefaultArgUnknownName.idr
2+${IDRIS:-idris} $@ --nocolour --check DefaultArgUnknownName.idr
33rm -f *.ibc
test/proof008/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --check $@ ClaimAndUnfocus.idr
2+${IDRIS:-idris} $@ --nocolour --check ClaimAndUnfocus.idr
33rm -f *.ibc
test/proofsearch003/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --check proofsearch003.idr
2+${IDRIS:-idris} $@ --check proofsearch003.idr
33rm -f *.ibc
test/reg002/run+1−1
@@ -1,4 +1,4 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} reg012.idr -o reg012
2+${IDRIS:-idris} $@ reg012.idr -o reg012
33./reg012
44rm -f reg012 *.ibc
test/reg003/run+2−2
@@ -1,4 +1,4 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --check reg003.idr
3−${IDRIS:-idris} --check reg003a.idr
2+${IDRIS:-idris} $@ --check reg003.idr
3+${IDRIS:-idris} $@ --check reg003a.idr
44rm -f *.ibc
test/reg007/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --check $@ reg007.lidr
2+${IDRIS:-idris} $@ --nocolour --check reg007.lidr
33rm -f *.ibc
test/reg023/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour $@ reg023.idr --check
2+${IDRIS:-idris} $@ --nocolour reg023.idr --check
33rm -f *.ibc
test/reg034/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --check reg034.idr
2+${IDRIS:-idris} $@ --nocolour --check reg034.idr
33rm -f reg034 *.ibc
test/reg035/run+3−3
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --check reg035.idr
3−${IDRIS:-idris} --nocolour --check reg035a.lidr
4−${IDRIS:-idris} --nocolour --check reg035b.idr
2+${IDRIS:-idris} $@ --nocolour --check reg035.idr
3+${IDRIS:-idris} $@ --nocolour --check reg035a.lidr
4+${IDRIS:-idris} $@ --nocolour --check reg035b.idr
55rm -f *.ibc
test/reg039/run+2−2
@@ -7,7 +7,7 @@ do
77 extraargs=("${extraargs[@]}" "'$arg'")
88done
99if (which timeout&>/dev/null); then
10− timeout 60 idris $@ --nocolour reg039.idr --exec go
10+ timeout 60 ${IDRIS:-idris} $@ --nocolour reg039.idr --exec go
1111else
1212 # From http://unix.stackexchange.com/questions/43340/how-to-introduce-timeout-for-shell-scripting
1313 # Executes command with a timeout
@@ -30,6 +30,6 @@ else
3030 fi
3131
3232 }
33− timeout 60 "idris ${extraargs[*]} --nocolour reg039.idr --exec go"
33+ timeout 60 "${IDRIS:-idris} ${extraargs[*]} --nocolour reg039.idr --exec go"
3434 fi
3535 rm -f reg039 *.ibc
test/reg040/run+1−1
@@ -1,6 +1,6 @@
11#!/usr/bin/env bash
22
3−${IDRIS:-idris} --warnreach reg040.idr -o reg040
3+${IDRIS:-idris} $@ --warnreach reg040.idr -o reg040
44./reg040
55
66rm -f reg040 *.ibc
test/reg044/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour reg044.idr --check
2+${IDRIS:-idris} $@ --nocolour reg044.idr --check
33rm -f *.ibc
test/reg049/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --check $@ reg049.idr
2+${IDRIS:-idris} $@ --nocolour --check $@ reg049.idr
33rm -f *.ibc
test/reg050/run+3−3
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour --check $@ working.idr
3−${IDRIS:-idris} --nocolour --check $@ badbangop.idr
4−${IDRIS:-idris} --nocolour --check $@ baddoublebang.idr
2+${IDRIS:-idris} $@ --nocolour --check working.idr
3+${IDRIS:-idris} $@ --nocolour --check badbangop.idr
4+${IDRIS:-idris} $@ --nocolour --check baddoublebang.idr
55rm -f *.ibc
test/reg054/run+1−1
@@ -1,2 +1,2 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour $@ reg054.idr --check
2+${IDRIS:-idris} $@ --nocolour reg054.idr --check
test/reg055/run+2−2
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour $@ reg055.idr --check
3−${IDRIS:-idris} --nocolour $@ reg055a.idr --check
2+${IDRIS:-idris} $@ --nocolour reg055.idr --check
3+${IDRIS:-idris} $@ --nocolour reg055a.idr --check
44rm -f *.ibc
55
test/reg075/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --quiet reg075.idr < input
2+${IDRIS:-idris} $@ --quiet reg075.idr < input
33rm -f *.ibc
test/runtest.hsdeleted−230
@@ -1,230 +0,0 @@
1−{-# LANGUAGE CPP #-}
2−module Main where
3−
4−import Control.Monad
5−import Data.Char
6−import Data.List
7−import Data.Maybe
8−import qualified Data.Set as S
9−import Data.Time.Clock
10−import System.Directory
11−import System.Environment
12−import System.FilePath
13−import System.Exit
14−import System.Info
15−import System.IO
16−import System.Process
17−
18−-- Because GHC earlier than 7.8 lacks setEnv
19−-- Install the setenv package on Windows.
20−#if __GLASGOW_HASKELL__ < 708
21−#ifndef mingw32_HOST_OS
22−import qualified System.Posix.Env as PE(setEnv)
23−
24−setEnv k v = PE.setEnv k v True
25−#else
26−import System.SetEnv(setEnv)
27−#endif
28−#endif
29−
30−data Flag = Update | Diff | ShowOutput | Quiet | Time deriving (Eq, Show, Ord)
31−
32−type Flags = S.Set Flag
33−
34−data Status = Success | Failure | Updated deriving (Eq, Show)
35−
36−data Config = Config {
37− flags :: Flags,
38− idrOpts :: [String],
39− tests :: [String]
40−} deriving (Show, Eq)
41−
42−isQuiet conf = Quiet `S.member` (flags conf)
43−showOutput conf = ShowOutput `S.member` (flags conf)
44−showTime conf = Time `S.member` (flags conf)
45−showDiff conf = Diff `S.member` (flags conf)
46−doUpdate conf = Update `S.member` (flags conf)
47−
48−checkTestName :: String -> Bool
49−checkTestName d = (all isDigit $ take 3 $ reverse d)
50− && (not $ isInfixOf "disabled" d)
51−
52−enumTests :: IO [String]
53−enumTests = do
54− cwd <- getCurrentDirectory
55− dirs <- getDirectoryContents cwd
56− return $ sort $ filter checkTestName dirs
57−
58−parseFlag :: String -> Maybe Flag
59−parseFlag s = case s of
60− "-u" -> Just Update
61− "-d" -> Just Diff
62− "-s" -> Just ShowOutput
63− "-t" -> Just Time
64− "-q" -> Just Quiet
65− _ -> Nothing
66−
67−parseFlags :: [String] -> (S.Set Flag, [String])
68−parseFlags xs = (S.fromList f, i)
69− where
70− f = catMaybes $ map parseFlag fl
71− (fl, i) = partition (\s -> parseFlag s /= Nothing) xs
72−
73−parseArgs :: [String] -> IO Config
74−parseArgs args = do
75− (tests, rest) <- case args of
76− ("all":xs) -> do
77− et <- enumTests
78− return (et, xs)
79− ("without":xs) -> do
80− t <- enumTests
81− (blacklist, ys) <- return $ break (== "opts") xs
82− return (t \\ blacklist, ys \\ ["opts"])
83− (x:xs) -> do
84− exists <- doesDirectoryExist x
85− return (if checkTestName x && exists then [x] else [], xs)
86− [] -> do
87− et <- enumTests
88− return (et, [])
89− let (testOpts, idOpts) = parseFlags rest
90− return $ Config testOpts idOpts tests
91−
92−-- "bash" needed because Haskell has cmd as the default shell on windows, and
93−-- we also want to run the process with another current directory, so we get
94−-- this thing.
95−runInShell :: String -> [String] -> IO (ExitCode, String)
96−runInShell test opts = do
97− (ec, output, _) <- readCreateProcessWithExitCode
98− ((proc "bash" ("run":opts)) { cwd = Just test,
99− std_out = CreatePipe })
100− ""
101− return (ec, output)
102−
103−runTest :: Config -> String -> IO Status
104−runTest conf test = do
105− -- don't touch the current directory as we want to run these things
106− -- in parallel in the future
107− let inTest s = test ++ "/" ++ s
108− t1 <- getCurrentTime
109− (exitCode, output) <- runInShell test (idrOpts conf)
110− t2 <- getCurrentTime
111− expected <- readFile $ inTest "expected"
112− writeFile (inTest "output") output
113− res <- if (norm output == norm expected)
114− then do putStrLn $ test ++ " finished...success"
115− return Success
116− else if doUpdate conf
117− then do putStrLn $ test ++ " finished...UPDATE"
118− writeFile (inTest "expected") output
119− return Updated
120− else do putStrLn $ test ++ " finished...FAILURE"
121− _ <- rawSystem "diff" [inTest "output", inTest "expected"]
122− return Failure
123− when (showTime conf) $ do
124− let dt = diffUTCTime t2 t1
125− putStrLn $ "Duration of " ++ test ++ " was " ++ show dt
126− return res
127− where
128− -- just pretend that backslashes are slashes for comparison
129− -- purposes to avoid path problems, so don't write any tests
130− -- that depend on that distinction in other contexts.
131− -- Also rewrite newlines for consistency.
132− norm ('\r':'\n':xs) = '\n' : norm xs
133− norm ('\\':'\\':xs) = '/' : norm xs
134− norm ('\\':xs) = '/' : norm xs
135− norm (x : xs) = x : norm xs
136− norm [] = []
137−
138−printStats :: Config -> [Status] -> IO ()
139−printStats conf stats = do
140− let total = length stats
141− let successful = length $ filter (== Success) stats
142− let failures = length $ filter (== Failure) stats
143− let updates = length $ filter (== Updated) stats
144− putStrLn "\n----"
145− putStrLn $ show total ++ " tests run: " ++ show successful ++ " succesful, "
146− ++ show failures ++ " failed, " ++ show updates ++ " updated."
147− let failed = map fst $ filter ((== Failure) . snd) $ zip (tests conf) stats
148− when (failed /= []) $ do
149− putStrLn "\nFailed tests:"
150− mapM_ putStrLn failed
151− putStrLn ""
152−
153−runTests :: Config -> IO Bool
154−runTests conf = do
155− stats <- mapM (runTest conf) (tests conf)
156− unless (isQuiet conf) $ printStats conf stats
157− return $ all (== Success) stats
158−
159−runShow :: Config -> IO Bool
160−runShow conf = do
161− mapM_ (\t -> callProcess "cat" [t++"/output"]) (tests conf)
162− return True
163−
164−runDiff :: Config -> IO Bool
165−runDiff conf = do
166− mapM_ (\t -> do putStrLn $ "Differences in " ++ t ++ ":"
167− ec <- rawSystem "diff" [t++"/output", t++"/expected"]
168− when (ec == ExitSuccess) $ putStrLn "No differences found.")
169− (tests conf)
170− return True
171−
172−whisper :: Config -> String -> IO ()
173−whisper conf s = do unless (isQuiet conf) $ putStrLn s
174−
175−isWindows :: Bool
176−isWindows = os `elem` ["win32", "mingw32", "cygwin32"]
177−
178−setPath :: Config -> IO ()
179−setPath conf = do
180− maybeEnv <- lookupEnv "IDRIS"
181− idrisExists <- case maybeEnv of
182− Just idrisExe -> do
183− let exeExtension = if isWindows then ".exe" else ""
184− doesFileExist (idrisExe ++ exeExtension)
185− Nothing -> return False
186− if (idrisExists)
187− then do
188− idrisAbs <- makeAbsolute $ fromMaybe "" maybeEnv
189− setEnv "IDRIS" idrisAbs
190− whisper conf $ "Using " ++ idrisAbs
191− else do
192− path <- getEnv "PATH"
193− setEnv "IDRIS" ""
194− let sandbox = "../.cabal-sandbox/bin"
195− hasBox <- doesDirectoryExist sandbox
196− bindir <- if hasBox
197− then do
198− whisper conf $ "Using Cabal sandbox at " ++ sandbox
199− makeAbsolute sandbox
200− else do
201− stackExe <- findExecutable "stack"
202− case stackExe of
203− Just stack -> do
204− out <- readProcess stack ["path", "--dist-dir"] []
205− stackDistDir <- return $ takeWhile (/= '\n') out
206− let stackDir = "../" ++ stackDistDir ++ "/build/idris"
207− whisper conf $ "Using stack work dir at " ++ stackDir
208− makeAbsolute stackDir
209− Nothing -> return ""
210− when (bindir /= "") $ setEnv "PATH" (bindir ++ [searchPathSeparator] ++ path)
211−
212−main = do
213− hSetBuffering stdout LineBuffering
214− withCabal <- doesDirectoryExist "test"
215− when withCabal $ do
216− setCurrentDirectory "test"
217− args <- getArgs
218− conf <- parseArgs args
219− setPath conf
220− t1 <- getCurrentTime
221− res <- case tests conf of
222− [] -> return True
223− xs | showOutput conf -> runShow conf
224− xs | showDiff conf -> runDiff conf
225− xs -> runTests conf
226− t2 <- getCurrentTime
227− when (showTime conf) $ do
228− let dt = diffUTCTime t2 t1
229− putStrLn $ "Duration of Entire Test Suite was " ++ show dt
230− unless res exitFailure
test/totality003/run+2−2
@@ -1,4 +1,4 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour $@ totality003.idr --check
3−${IDRIS:-idris} --nocolour $@ totality003a.idr --check
2+${IDRIS:-idris} $@ --nocolour totality003.idr --check
3+${IDRIS:-idris} $@ --nocolour totality003a.idr --check
44rm -f *.ibc
test/totality009/run+3−3
@@ -1,6 +1,6 @@
11#!/usr/bin/env bash
22OPTS="--consolewidth infinite --nocolour"
3−${IDRIS:-idris} "$@" $OPTS --check TestLambdaImpossible
4−${IDRIS:-idris} "$@" $OPTS --check --warnpartial TestLambdaPossible
5−${IDRIS:-idris} "$@" $OPTS --check TestLambdaPossible2
3+${IDRIS:-idris} $@ $OPTS --check TestLambdaImpossible
4+${IDRIS:-idris} $@ $OPTS --check --warnpartial TestLambdaPossible
5+${IDRIS:-idris} $@ $OPTS --check TestLambdaPossible2
66rm -f *.ibc
test/tutorial001/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --check tutorial001.idr
2+${IDRIS:-idris} $@ --check tutorial001.idr
33rm -f *.ibc
test/tutorial006/run+3−3
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour $@ tutorial006a.idr --check
3−${IDRIS:-idris} --nocolour $@ tutorial006b.idr --check
4−${IDRIS:-idris} --nocolour $@ tutorial006c.idr -p effects --check
2+${IDRIS:-idris} $@ --nocolour tutorial006a.idr --check
3+${IDRIS:-idris} $@ --nocolour tutorial006b.idr --check
4+${IDRIS:-idris} $@ --nocolour tutorial006c.idr -p effects --check
55rm -f *.ibc
test/tutorial007/run+2−2
@@ -1,5 +1,5 @@
11#!/usr/bin/env bash
2−gcc -shared -fPIC nativetypes.c -o nativetypes.so
3−idris $@ tutorial007.idr -o tutorial007
2+${CC:-cc} -shared -fPIC nativetypes.c -o nativetypes.so
3+${IDRIS:-idris} $@ tutorial007.idr -o tutorial007
44./tutorial007
55rm -f tutorial007 *.ibc *.so sizefromc.txt
test/unique001/run+6−6
@@ -1,9 +1,9 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour $@ unique001.idr -o unique001
2+${IDRIS:-idris} $@ --nocolour unique001.idr -o unique001
33./unique001
4−${IDRIS:-idris} --nocolour $@ unique001a.idr --check
5−${IDRIS:-idris} --nocolour $@ unique001b.idr --check
6−${IDRIS:-idris} --nocolour $@ unique001c.idr --check
7−${IDRIS:-idris} --nocolour $@ unique001d.idr --check
8−${IDRIS:-idris} --nocolour $@ unique001e.idr --check
4+${IDRIS:-idris} $@ --nocolour unique001a.idr --check
5+${IDRIS:-idris} $@ --nocolour unique001b.idr --check
6+${IDRIS:-idris} $@ --nocolour unique001c.idr --check
7+${IDRIS:-idris} $@ --nocolour unique001d.idr --check
8+${IDRIS:-idris} $@ --nocolour unique001e.idr --check
99rm -f unique001 *.ibc
test/unique002/run+2−2
@@ -1,4 +1,4 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour $@ unique002.idr --check
3−${IDRIS:-idris} --nocolour $@ unique002a.idr --check
2+${IDRIS:-idris} $@ --nocolour unique002.idr --check
3+${IDRIS:-idris} $@ --nocolour unique002a.idr --check
44rm -f *.ibc
test/unique003/run+1−1
@@ -1,3 +1,3 @@
11#!/usr/bin/env bash
2−${IDRIS:-idris} --nocolour $@ unique003.idr --check
2+${IDRIS:-idris} $@ --nocolour unique003.idr --check
33rm -f *.ibc