mirror of
https://github.com/kanaka/mal.git
synced 2024-09-11 13:55:55 +03:00
runtest.py: add --soft option, refactor test reader.
- also, mark later stepA optional as soft.
This commit is contained in:
Joel Martin
parent
fdf80511d4
commit
98af2ae360
126
runtest.py
126
runtest.py
@ -31,6 +31,8 @@ parser.add_argument('--no-pty', action='store_true',
|
||||
help="Use direct pipes instead of pseudo-tty")
|
||||
parser.add_argument('--log-file', type=str,
|
||||
help="Write all test interaction the named file")
|
||||
parser.add_argument('--soft', action='store_true',
|
||||
help="Report but do not fail tests after ';>>> soft=True'")
|
||||
|
||||
parser.add_argument('test_file', type=argparse.FileType('r'),
|
||||
help="a test file formatted as with mal test data")
|
||||
@ -130,67 +132,79 @@ class Runner():
|
||||
pass
|
||||
self.p = None
|
||||
|
||||
class TestReader:
|
||||
def __init__(self, test_file):
|
||||
self.line_num = 0
|
||||
self.data = test_file.read().split('\n')
|
||||
self.soft = False
|
||||
|
||||
def next(self):
|
||||
self.form = None
|
||||
self.out = ""
|
||||
self.ret = None
|
||||
|
||||
while self.data:
|
||||
self.line_num += 1
|
||||
line = self.data.pop(0)
|
||||
if re.match(r"^\s*$", line): # blank line
|
||||
continue
|
||||
elif line[0:3] == ";;;": # ignore comment
|
||||
continue
|
||||
elif line[0:2] == ";;": # output comment
|
||||
print(line[3:])
|
||||
continue
|
||||
elif line[0:5] == ";>>> ": # settings/commands
|
||||
settings = {}
|
||||
exec(line[5:], {}, settings)
|
||||
if 'soft' in settings: self.soft = True
|
||||
continue
|
||||
elif line[0:1] == ";": # unexpected comment
|
||||
print("Test data error at line %d:\n%s" % (self.line_num, line))
|
||||
return None
|
||||
self.form = line # the line is a form to send
|
||||
|
||||
# Now find the output and return value
|
||||
while self.data:
|
||||
line = self.data[0]
|
||||
if line[0:3] == ";=>":
|
||||
self.ret = line[3:].replace('\\r', '\r').replace('\\n', '\n')
|
||||
self.line_num += 1
|
||||
self.data.pop(0)
|
||||
break
|
||||
elif line[0:2] == "; ":
|
||||
self.out = self.out + line[2:] + sep
|
||||
self.line_num += 1
|
||||
self.data.pop(0)
|
||||
else:
|
||||
self.ret = "*"
|
||||
break
|
||||
if self.ret: break
|
||||
|
||||
return self.form
|
||||
|
||||
|
||||
args = parser.parse_args(sys.argv[1:])
|
||||
test_data = args.test_file.read().split('\n')
|
||||
|
||||
if args.rundir: os.chdir(args.rundir)
|
||||
|
||||
r = Runner(args.mal_cmd, no_pty=args.no_pty, log_file=args.log_file)
|
||||
t = TestReader(args.test_file)
|
||||
|
||||
|
||||
test_idx = 0
|
||||
def read_test(data):
|
||||
global test_idx
|
||||
form, output, ret = None, "", None
|
||||
while data:
|
||||
test_idx += 1
|
||||
line = data.pop(0)
|
||||
if re.match(r"^\s*$", line): # blank line
|
||||
continue
|
||||
elif line[0:3] == ";;;": # ignore comment
|
||||
continue
|
||||
elif line[0:2] == ";;": # output comment
|
||||
print(line[3:])
|
||||
continue
|
||||
elif line[0:2] == ";": # unexpected comment
|
||||
print("Test data error at line %d:\n%s" % (test_idx, line))
|
||||
return None, None, None, test_idx
|
||||
form = line # the line is a form to send
|
||||
|
||||
# Now find the output and return value
|
||||
while data:
|
||||
line = data[0]
|
||||
if line[0:3] == ";=>":
|
||||
ret = line[3:].replace('\\r', '\r').replace('\\n', '\n')
|
||||
test_idx += 1
|
||||
data.pop(0)
|
||||
break
|
||||
elif line[0:2] == "; ":
|
||||
output = output + line[2:] + sep
|
||||
test_idx += 1
|
||||
data.pop(0)
|
||||
else:
|
||||
ret = "*"
|
||||
break
|
||||
if ret: break
|
||||
|
||||
return form, output, ret, test_idx
|
||||
|
||||
def assert_prompt(timeout):
|
||||
def assert_prompt(runner, prompts, timeout):
|
||||
# Wait for the initial prompt
|
||||
header = r.read_to_prompt(['user> ', 'mal-user> '], timeout=timeout)
|
||||
header = runner.read_to_prompt(prompts, timeout=timeout)
|
||||
if not header == None:
|
||||
if header:
|
||||
print("Started with:\n%s" % header)
|
||||
else:
|
||||
print("Did not get 'user> ' or 'mal-user> ' prompt")
|
||||
print("Did not one of following prompt(s): %s" % repr(prompts))
|
||||
print(" Got : %s" % repr(r.buf))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# Wait for the initial prompt
|
||||
assert_prompt(args.start_timeout)
|
||||
assert_prompt(r, ['user> ', 'mal-user> '], args.start_timeout)
|
||||
|
||||
# Send the pre-eval code if any
|
||||
if args.pre_eval:
|
||||
@ -199,39 +213,43 @@ if args.pre_eval:
|
||||
assert_prompt(args.test_timeout)
|
||||
|
||||
fail_cnt = 0
|
||||
soft_fail_cnt = 0
|
||||
|
||||
while test_data:
|
||||
form, out, ret, line_num = read_test(test_data)
|
||||
if form == None:
|
||||
break
|
||||
sys.stdout.write("TEST: %s -> [%s,%s]" % (form, repr(out), repr(ret)))
|
||||
while t.next():
|
||||
sys.stdout.write("TEST: %s -> [%s,%s]" % (t.form, repr(t.out), t.ret))
|
||||
sys.stdout.flush()
|
||||
|
||||
# The repeated form is to get around an occasional OS X issue
|
||||
# where the form is repeated.
|
||||
# https://github.com/kanaka/mal/issues/30
|
||||
expected = ["%s%s%s%s" % (form, sep, out, ret),
|
||||
"%s%s%s%s%s%s" % (form, sep, form, sep, out, ret)]
|
||||
expected = ["%s%s%s%s" % (t.form, sep, t.out, t.ret),
|
||||
"%s%s%s%s%s%s" % (t.form, sep, t.form, sep, t.out, t.ret)]
|
||||
|
||||
r.writeline(form)
|
||||
r.writeline(t.form)
|
||||
try:
|
||||
res = r.read_to_prompt(['\r\nuser> ', '\nuser> ',
|
||||
'\r\nmal-user> ', '\nmal-user> '],
|
||||
timeout=args.test_timeout)
|
||||
#print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
|
||||
if ret == "*" or res in expected:
|
||||
if t.ret == "*" or res in expected:
|
||||
print(" -> SUCCESS")
|
||||
else:
|
||||
print(" -> FAIL (line %d):" % line_num)
|
||||
if args.soft and t.soft:
|
||||
print(" -> SOFT FAIL (line %d):" % t.line_num)
|
||||
soft_fail_cnt += 1
|
||||
else:
|
||||
print(" -> FAIL (line %d):" % t.line_num)
|
||||
fail_cnt += 1
|
||||
print(" Expected : %s" % repr(expected))
|
||||
print(" Got : %s" % repr(res))
|
||||
fail_cnt += 1
|
||||
except:
|
||||
_, exc, _ = sys.exc_info()
|
||||
print("\nException: %s" % repr(exc))
|
||||
print("Output before exception:\n%s" % r.buf)
|
||||
sys.exit(1)
|
||||
|
||||
if soft_fail_cnt > 0:
|
||||
print("SOFT FAILURES: %d" % soft_fail_cnt)
|
||||
if fail_cnt > 0:
|
||||
print("FAILURES: %d" % fail_cnt)
|
||||
sys.exit(2)
|
||||
|
||||
@ -143,6 +143,7 @@
|
||||
;;
|
||||
;; ------- Optional Functionality --------------
|
||||
;; ------- (Not needed for self-hosting) -------
|
||||
;>>> soft=True
|
||||
|
||||
;;
|
||||
;; Testing conj function
|
||||
|
||||
Loading…
Reference in New Issue
Block a user