mirror of
https://github.com/kanaka/mal.git
synced 2024-08-17 01:30:26 +03:00
runtest.py: order of magnitude faster.
C#, VB and Lua tests are broken though.
This commit is contained in:
parent
c2a6838f39
commit
7907cd904a
11
Makefile
11
Makefile
@ -29,18 +29,21 @@ stepA = stepA_mal
|
||||
EXCLUDE_TESTS += test^bash^step5 # no stack exhaustion or completion
|
||||
EXCLUDE_TESTS += test^c^step5 # segfault
|
||||
EXCLUDE_TESTS += test^cs^step5 # fatal stack overflow fault
|
||||
EXCLUDE_TESTS += test^haskell^step5 # test completes
|
||||
EXCLUDE_TESTS += test^make^step5 # no TCO capability/step
|
||||
EXCLUDE_TESTS += test^mal^step5 # no TCO capability/step
|
||||
EXCLUDE_TESTS += test^go^step5 # test completes, even at 100,000
|
||||
EXCLUDE_TESTS += test^php^step5 # test completes, even at 100,000
|
||||
EXCLUDE_TESTS += test^racket^step5 # test completes
|
||||
EXCLUDE_TESTS += test^ruby^step5 # test completes, even at 100,000
|
||||
EXCLUDE_TESTS += test^rust^step5 # no catching stack overflows
|
||||
EXCLUDE_TESTS += test^ocaml^step5 # test completes, even at 1,000,000
|
||||
|
||||
# interop tests now implemented yet
|
||||
EXCLUDE_TESTS += test^cs^stepA test^java^stepA test^mal^stepA \
|
||||
test^mal^step0 test^php^stepA test^ps^stepA \
|
||||
test^python^stepA test^ruby^stepA
|
||||
EXCLUDE_TESTS += test^cs^stepA test^go^stepA test^haskell^stepA \
|
||||
test^java^stepA test^mal^stepA test^mal^step0 \
|
||||
test^php^stepA test^ps^stepA test^python^stepA \
|
||||
test^ruby^stepA
|
||||
|
||||
EXCLUDE_PERFS = perf^mal # TODO: fix this
|
||||
|
||||
@ -112,7 +115,7 @@ vb_RUNSTEP = mono ../$(2) --raw $(3)
|
||||
|
||||
# Extra options to pass to runtest.py
|
||||
cs_TEST_OPTS = --redirect
|
||||
mal_TEST_OPTS = --redirect --start-timeout 60 --test-timeout 120
|
||||
mal_TEST_OPTS = --start-timeout 60 --test-timeout 120
|
||||
vb_TEST_OPTS = --redirect
|
||||
|
||||
|
||||
|
@ -9,8 +9,8 @@ All:
|
||||
of iterations per second
|
||||
- redefine (defmacro!) as (def! (macro*))
|
||||
- runtest expect fixes:
|
||||
- stop using expect, so we can drop --raw option
|
||||
- fix long lines in runtext/expect
|
||||
* stop using expect, so we can drop --raw option
|
||||
- fix long line splitting in runtest
|
||||
- regular expression matching in runtest
|
||||
- add re (use in rep) everywhere and use that (to avoid printing)
|
||||
- Implement/fix interop
|
||||
|
134
runtest-old.py
Executable file
134
runtest-old.py
Executable file
@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os, sys, re
|
||||
import argparse
|
||||
|
||||
# http://pexpect.sourceforge.net/pexpect.html
|
||||
from pexpect import spawn, EOF, TIMEOUT
|
||||
|
||||
# TODO: do we need to support '\n' too
|
||||
sep = "\r\n"
|
||||
rundir = None
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run a test file against a Mal implementation")
|
||||
parser.add_argument('--rundir',
|
||||
help="change to the directory before running tests")
|
||||
parser.add_argument('--start-timeout', default=10, type=int,
|
||||
help="default timeout for initial prompt")
|
||||
parser.add_argument('--test-timeout', default=20, type=int,
|
||||
help="default timeout for each individual test action")
|
||||
parser.add_argument('--pre-eval', default=None, type=str,
|
||||
help="Mal code to evaluate prior to running the test")
|
||||
parser.add_argument('--redirect', action='store_true',
|
||||
help="Run implementation in bash and redirect output to /dev/null")
|
||||
|
||||
parser.add_argument('test_file', type=argparse.FileType('r'),
|
||||
help="a test file formatted as with mal test data")
|
||||
parser.add_argument('mal_cmd', nargs="*",
|
||||
help="Mal implementation command line. Use '--' to "
|
||||
"specify a Mal command line with dashed options.")
|
||||
|
||||
args = parser.parse_args(sys.argv[1:])
|
||||
test_data = args.test_file.read().split('\n')
|
||||
|
||||
if args.rundir: os.chdir(args.rundir)
|
||||
|
||||
if args.redirect:
|
||||
# Redirect to try and force raw mode (no ASCII codes)
|
||||
p = spawn('/bin/bash -c "' + " ".join(args.mal_cmd) + ' |tee /dev/null"')
|
||||
else:
|
||||
p = spawn(args.mal_cmd[0], args.mal_cmd[1:])
|
||||
|
||||
|
||||
test_idx = 0
|
||||
def read_test(data):
|
||||
global test_idx
|
||||
form, output, ret = None, "", None
|
||||
while data:
|
||||
test_idx += 1
|
||||
line = data.pop(0)
|
||||
if re.match(r"^\s*$", line): # blank line
|
||||
continue
|
||||
elif line[0:3] == ";;;": # ignore comment
|
||||
continue
|
||||
elif line[0:2] == ";;": # output comment
|
||||
print line[3:]
|
||||
continue
|
||||
elif line[0:2] == ";": # unexpected comment
|
||||
print "Test data error at line %d:\n%s" % (test_idx, line)
|
||||
return None, None, None, test_idx
|
||||
form = line # the line is a form to send
|
||||
|
||||
# Now find the output and return value
|
||||
while data:
|
||||
line = data[0]
|
||||
if line[0:3] == ";=>":
|
||||
ret = line[3:].replace('\\r', '\r').replace('\\n', '\n')
|
||||
test_idx += 1
|
||||
data.pop(0)
|
||||
break
|
||||
elif line[0:2] == "; ":
|
||||
output = output + line[2:] + sep
|
||||
test_idx += 1
|
||||
data.pop(0)
|
||||
else:
|
||||
ret = "*"
|
||||
break
|
||||
if ret: break
|
||||
|
||||
return form, output, ret, test_idx
|
||||
|
||||
def assert_prompt(timeout):
|
||||
# Wait for the initial prompt
|
||||
idx = p.expect(['user> ', 'mal-user> ', EOF, TIMEOUT],
|
||||
timeout=timeout)
|
||||
if idx not in [0,1]:
|
||||
print "Did not get 'user> ' or 'mal-user> ' prompt"
|
||||
print " Got : %s" % repr(p.before)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# Wait for the initial prompt
|
||||
assert_prompt(args.start_timeout)
|
||||
|
||||
# Send the pre-eval code if any
|
||||
if args.pre_eval:
|
||||
sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval)
|
||||
p.sendline(args.pre_eval)
|
||||
assert_prompt(args.test_timeout)
|
||||
|
||||
fail_cnt = 0
|
||||
|
||||
while test_data:
|
||||
form, out, ret, line_num = read_test(test_data)
|
||||
if form == None:
|
||||
break
|
||||
sys.stdout.write("TEST: %s -> [%s,%s]" % (form, repr(out), repr(ret)))
|
||||
sys.stdout.flush()
|
||||
expected = "%s%s%s%s" % (form, sep, out, ret)
|
||||
|
||||
p.sendline(form)
|
||||
try:
|
||||
idx = p.expect(['\r\nuser> ', '\nuser> ',
|
||||
'\r\nmal-user> ', '\nmal-user> '],
|
||||
timeout=args.test_timeout)
|
||||
#print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
|
||||
if ret == "*" or p.before == expected:
|
||||
print " -> SUCCESS"
|
||||
else:
|
||||
print " -> FAIL (line %d):" % line_num
|
||||
print " Expected : %s" % repr(expected)
|
||||
print " Got : %s" % repr(p.before)
|
||||
fail_cnt += 1
|
||||
except EOF:
|
||||
print "Got EOF"
|
||||
sys.exit(1)
|
||||
except TIMEOUT:
|
||||
print "Got TIMEOUT, received: %s" % repr(p.before)
|
||||
sys.exit(1)
|
||||
|
||||
if fail_cnt > 0:
|
||||
print "FAILURES: %d" % fail_cnt
|
||||
sys.exit(2)
|
||||
sys.exit(0)
|
86
runtest.py
86
runtest.py
@ -1,13 +1,15 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os, sys, re
|
||||
import argparse
|
||||
import argparse, time
|
||||
|
||||
# http://pexpect.sourceforge.net/pexpect.html
|
||||
from pexpect import spawn, EOF, TIMEOUT
|
||||
import pty
|
||||
from subprocess import Popen, STDOUT, PIPE
|
||||
from select import select
|
||||
|
||||
# TODO: do we need to support '\n' too
|
||||
sep = "\r\n"
|
||||
#sep = "\n"
|
||||
rundir = None
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
@ -29,16 +31,53 @@ parser.add_argument('mal_cmd', nargs="*",
|
||||
help="Mal implementation command line. Use '--' to "
|
||||
"specify a Mal command line with dashed options.")
|
||||
|
||||
class Runner():
|
||||
def __init__(self, args, redirect=False):
|
||||
print "args: %s" % repr(args)
|
||||
if redirect:
|
||||
print "using redirect"
|
||||
self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
|
||||
self.stdin = self.p.stdin
|
||||
self.stdout = self.p.stdout
|
||||
else:
|
||||
# provide tty to get 'interactive' readline to work
|
||||
master, slave = pty.openpty()
|
||||
self.p = Popen(args, bufsize=0, stdin=slave, stdout=slave, stderr=STDOUT)
|
||||
self.stdin = os.fdopen(master, 'r+b', 0)
|
||||
self.stdout = self.stdin
|
||||
|
||||
#print "started"
|
||||
self.buf = ""
|
||||
self.last_prompt = ""
|
||||
|
||||
def read_to_prompt(self, prompts, timeout):
|
||||
end_time = time.time() + timeout
|
||||
while time.time() < end_time:
|
||||
[outs,_,_] = select([self.stdin], [], [], 1)
|
||||
if self.stdin in outs:
|
||||
new_data = self.stdin.read(1)
|
||||
#print "new_data: '%s'" % new_data
|
||||
self.buf += new_data
|
||||
for prompt in prompts:
|
||||
regexp = re.compile(prompt)
|
||||
match = regexp.search(self.buf)
|
||||
if match:
|
||||
end = match.end()
|
||||
buf = self.buf[0:end-len(prompt)]
|
||||
self.buf = self.buf[end:]
|
||||
self.last_prompt = prompt
|
||||
return buf
|
||||
return None
|
||||
|
||||
def write(self, str):
|
||||
self.stdout.write(str)
|
||||
|
||||
args = parser.parse_args(sys.argv[1:])
|
||||
test_data = args.test_file.read().split('\n')
|
||||
|
||||
if args.rundir: os.chdir(args.rundir)
|
||||
|
||||
if args.redirect:
|
||||
# Redirect to try and force raw mode (no ASCII codes)
|
||||
p = spawn('/bin/bash -c "' + " ".join(args.mal_cmd) + ' |tee /dev/null"')
|
||||
else:
|
||||
p = spawn(args.mal_cmd[0], args.mal_cmd[1:])
|
||||
r = Runner(args.mal_cmd, redirect=args.redirect)
|
||||
|
||||
|
||||
test_idx = 0
|
||||
@ -81,11 +120,13 @@ def read_test(data):
|
||||
|
||||
def assert_prompt(timeout):
|
||||
# Wait for the initial prompt
|
||||
idx = p.expect(['user> ', 'mal-user> ', EOF, TIMEOUT],
|
||||
timeout=timeout)
|
||||
if idx not in [0,1]:
|
||||
header = r.read_to_prompt(['user> ', 'mal-user> '], timeout=timeout)
|
||||
if not header == None:
|
||||
if header:
|
||||
print "Started with:\n%s" % header
|
||||
else:
|
||||
print "Did not get 'user> ' or 'mal-user> ' prompt"
|
||||
print " Got : %s" % repr(p.before)
|
||||
print " Got : %s" % repr(r.buf)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@ -95,7 +136,7 @@ assert_prompt(args.start_timeout)
|
||||
# Send the pre-eval code if any
|
||||
if args.pre_eval:
|
||||
sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval)
|
||||
p.sendline(args.pre_eval)
|
||||
p.write(args.pre_eval)
|
||||
assert_prompt(args.test_timeout)
|
||||
|
||||
fail_cnt = 0
|
||||
@ -108,24 +149,21 @@ while test_data:
|
||||
sys.stdout.flush()
|
||||
expected = "%s%s%s%s" % (form, sep, out, ret)
|
||||
|
||||
p.sendline(form)
|
||||
r.write(form + "\n")
|
||||
try:
|
||||
idx = p.expect(['\r\nuser> ', '\nuser> ',
|
||||
'\r\nmal-user> ', '\nmal-user> '],
|
||||
timeout=args.test_timeout)
|
||||
res = r.read_to_prompt(['\r\nuser> ', '\nuser> ',
|
||||
'\r\nmal-user> ', '\nmal-user> '],
|
||||
timeout=args.test_timeout)
|
||||
#print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
|
||||
if ret == "*" or p.before == expected:
|
||||
if ret == "*" or res == expected:
|
||||
print " -> SUCCESS"
|
||||
else:
|
||||
print " -> FAIL (line %d):" % line_num
|
||||
print " Expected : %s" % repr(expected)
|
||||
print " Got : %s" % repr(p.before)
|
||||
print " Got : %s" % repr(res)
|
||||
fail_cnt += 1
|
||||
except EOF:
|
||||
print "Got EOF"
|
||||
sys.exit(1)
|
||||
except TIMEOUT:
|
||||
print "Got TIMEOUT, received: %s" % repr(p.before)
|
||||
except:
|
||||
print "Got Exception"
|
||||
sys.exit(1)
|
||||
|
||||
if fail_cnt > 0:
|
||||
|
Loading…
Reference in New Issue
Block a user