kanaka/mal
1
1
Fork 0
mirror of https://github.com/kanaka/mal.git synced 2024-08-17 01:30:26 +03:00
Code Issues Projects Releases Wiki Activity

runtest.py: order of magnitude faster.

Browse Source 
C#, VB and Lua tests are broken though.
This commit is contained in:
Joel Martin 2015-02-28 15:41:18 -06:00
parent c2a6838f39
commit 7907cd904a
4 changed files with 205 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Makefile
View File

@ -29,18 +29,21 @@ stepA = stepA_mal
EXCLUDE_TESTS += test^bash^step5 # no stack exhaustion or completion
EXCLUDE_TESTS += test^c^step5 # segfault
EXCLUDE_TESTS += test^cs^step5 # fatal stack overflow fault
EXCLUDE_TESTS += test^haskell^step5 # test completes
EXCLUDE_TESTS += test^make^step5 # no TCO capability/step
EXCLUDE_TESTS += test^mal^step5 # no TCO capability/step
EXCLUDE_TESTS += test^go^step5 # test completes, even at 100,000
EXCLUDE_TESTS += test^php^step5 # test completes, even at 100,000
EXCLUDE_TESTS += test^racket^step5 # test completes
EXCLUDE_TESTS += test^ruby^step5 # test completes, even at 100,000
EXCLUDE_TESTS += test^rust^step5 # no catching stack overflows
EXCLUDE_TESTS += test^ocaml^step5 # test completes, even at 1,000,000
# interop tests now implemented yet
EXCLUDE_TESTS += test^cs^stepA test^java^stepA test^mal^stepA \
test^mal^step0 test^php^stepA test^ps^stepA \
test^python^stepA test^ruby^stepA
EXCLUDE_TESTS += test^cs^stepA test^go^stepA test^haskell^stepA \
test^java^stepA test^mal^stepA test^mal^step0 \
test^php^stepA test^ps^stepA test^python^stepA \
test^ruby^stepA
EXCLUDE_PERFS = perf^mal # TODO: fix this
@ -112,7 +115,7 @@ vb_RUNSTEP = mono ../$(2) --raw $(3)
# Extra options to pass to runtest.py
cs_TEST_OPTS = --redirect
mal_TEST_OPTS = --redirect --start-timeout 60 --test-timeout 120
mal_TEST_OPTS = --start-timeout 60 --test-timeout 120
vb_TEST_OPTS = --redirect

4
docs/TODO
View File

@ -9,8 +9,8 @@ All:
of iterations per second
- redefine (defmacro!) as (def! (macro*))
- runtest expect fixes:
- stop using expect, so we can drop --raw option
- fix long lines in runtext/expect
* stop using expect, so we can drop --raw option
- fix long line splitting in runtest
- regular expression matching in runtest
- add re (use in rep) everywhere and use that (to avoid printing)
- Implement/fix interop

134
runtest-old.py Executable file
View File

@ -0,0 +1,134 @@
#!/usr/bin/env python
import os, sys, re
import argparse
# http://pexpect.sourceforge.net/pexpect.html
from pexpect import spawn, EOF, TIMEOUT
# TODO: do we need to support '\n' too
sep = "\r\n"
rundir = None
parser = argparse.ArgumentParser(
description="Run a test file against a Mal implementation")
parser.add_argument('--rundir',
help="change to the directory before running tests")
parser.add_argument('--start-timeout', default=10, type=int,
help="default timeout for initial prompt")
parser.add_argument('--test-timeout', default=20, type=int,
help="default timeout for each individual test action")
parser.add_argument('--pre-eval', default=None, type=str,
help="Mal code to evaluate prior to running the test")
parser.add_argument('--redirect', action='store_true',
help="Run implementation in bash and redirect output to /dev/null")
parser.add_argument('test_file', type=argparse.FileType('r'),
help="a test file formatted as with mal test data")
parser.add_argument('mal_cmd', nargs="*",
help="Mal implementation command line. Use '--' to "
"specify a Mal command line with dashed options.")
args = parser.parse_args(sys.argv[1:])
test_data = args.test_file.read().split('\n')
if args.rundir: os.chdir(args.rundir)
if args.redirect:
# Redirect to try and force raw mode (no ASCII codes)
p = spawn('/bin/bash -c "' + " ".join(args.mal_cmd) + ' |tee /dev/null"')
else:
p = spawn(args.mal_cmd[0], args.mal_cmd[1:])
test_idx = 0
def read_test(data):
global test_idx
form, output, ret = None, "", None
while data:
test_idx += 1
line = data.pop(0)
if re.match(r"^\s*$", line): # blank line
continue
elif line[0:3] == ";;;": # ignore comment
continue
elif line[0:2] == ";;": # output comment
print line[3:]
continue
elif line[0:2] == ";": # unexpected comment
print "Test data error at line %d:\n%s" % (test_idx, line)
return None, None, None, test_idx
form = line # the line is a form to send
# Now find the output and return value
while data:
line = data[0]
if line[0:3] == ";=>":
ret = line[3:].replace('\\r', '\r').replace('\\n', '\n')
test_idx += 1
data.pop(0)
break
elif line[0:2] == "; ":
output = output + line[2:] + sep
test_idx += 1
data.pop(0)
else:
ret = "*"
break
if ret: break
return form, output, ret, test_idx
def assert_prompt(timeout):
# Wait for the initial prompt
idx = p.expect(['user> ', 'mal-user> ', EOF, TIMEOUT],
timeout=timeout)
if idx not in [0,1]:
print "Did not get 'user> ' or 'mal-user> ' prompt"
print " Got : %s" % repr(p.before)
sys.exit(1)
# Wait for the initial prompt
assert_prompt(args.start_timeout)
# Send the pre-eval code if any
if args.pre_eval:
sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval)
p.sendline(args.pre_eval)
assert_prompt(args.test_timeout)
fail_cnt = 0
while test_data:
form, out, ret, line_num = read_test(test_data)
if form == None:
break
sys.stdout.write("TEST: %s -> [%s,%s]" % (form, repr(out), repr(ret)))
sys.stdout.flush()
expected = "%s%s%s%s" % (form, sep, out, ret)
p.sendline(form)
try:
idx = p.expect(['\r\nuser> ', '\nuser> ',
'\r\nmal-user> ', '\nmal-user> '],
timeout=args.test_timeout)
#print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
if ret == "*" or p.before == expected:
print " -> SUCCESS"
else:
print " -> FAIL (line %d):" % line_num
print " Expected : %s" % repr(expected)
print " Got : %s" % repr(p.before)
fail_cnt += 1
except EOF:
print "Got EOF"
sys.exit(1)
except TIMEOUT:
print "Got TIMEOUT, received: %s" % repr(p.before)
sys.exit(1)
if fail_cnt > 0:
print "FAILURES: %d" % fail_cnt
sys.exit(2)
sys.exit(0)

86
runtest.py
View File

@ -1,13 +1,15 @@
#!/usr/bin/env python
import os, sys, re
import argparse
import argparse, time
# http://pexpect.sourceforge.net/pexpect.html
from pexpect import spawn, EOF, TIMEOUT
import pty
from subprocess import Popen, STDOUT, PIPE
from select import select
# TODO: do we need to support '\n' too
sep = "\r\n"
#sep = "\n"
rundir = None
parser = argparse.ArgumentParser(
@ -29,16 +31,53 @@ parser.add_argument('mal_cmd', nargs="*",
help="Mal implementation command line. Use '--' to "
"specify a Mal command line with dashed options.")
class Runner():
def __init__(self, args, redirect=False):
print "args: %s" % repr(args)
if redirect:
print "using redirect"
self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
self.stdin = self.p.stdin
self.stdout = self.p.stdout
else:
# provide tty to get 'interactive' readline to work
master, slave = pty.openpty()
self.p = Popen(args, bufsize=0, stdin=slave, stdout=slave, stderr=STDOUT)
self.stdin = os.fdopen(master, 'r+b', 0)
self.stdout = self.stdin
#print "started"
self.buf = ""
self.last_prompt = ""
def read_to_prompt(self, prompts, timeout):
end_time = time.time() + timeout
while time.time() < end_time:
[outs,_,_] = select([self.stdin], [], [], 1)
if self.stdin in outs:
new_data = self.stdin.read(1)
#print "new_data: '%s'" % new_data
self.buf += new_data
for prompt in prompts:
regexp = re.compile(prompt)
match = regexp.search(self.buf)
if match:
end = match.end()
buf = self.buf[0:end-len(prompt)]
self.buf = self.buf[end:]
self.last_prompt = prompt
return buf
return None
def write(self, str):
self.stdout.write(str)
args = parser.parse_args(sys.argv[1:])
test_data = args.test_file.read().split('\n')
if args.rundir: os.chdir(args.rundir)
if args.redirect:
# Redirect to try and force raw mode (no ASCII codes)
p = spawn('/bin/bash -c "' + " ".join(args.mal_cmd) + ' |tee /dev/null"')
else:
p = spawn(args.mal_cmd[0], args.mal_cmd[1:])
r = Runner(args.mal_cmd, redirect=args.redirect)
test_idx = 0
@ -81,11 +120,13 @@ def read_test(data):
def assert_prompt(timeout):
# Wait for the initial prompt
idx = p.expect(['user> ', 'mal-user> ', EOF, TIMEOUT],
timeout=timeout)
if idx not in [0,1]:
header = r.read_to_prompt(['user> ', 'mal-user> '], timeout=timeout)
if not header == None:
if header:
print "Started with:\n%s" % header
else:
print "Did not get 'user> ' or 'mal-user> ' prompt"
print " Got : %s" % repr(p.before)
print " Got : %s" % repr(r.buf)
sys.exit(1)
@ -95,7 +136,7 @@ assert_prompt(args.start_timeout)
# Send the pre-eval code if any
if args.pre_eval:
sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval)
p.sendline(args.pre_eval)
p.write(args.pre_eval)
assert_prompt(args.test_timeout)
fail_cnt = 0
@ -108,24 +149,21 @@ while test_data:
sys.stdout.flush()
expected = "%s%s%s%s" % (form, sep, out, ret)
p.sendline(form)
r.write(form + "\n")
try:
idx = p.expect(['\r\nuser> ', '\nuser> ',
'\r\nmal-user> ', '\nmal-user> '],
timeout=args.test_timeout)
res = r.read_to_prompt(['\r\nuser> ', '\nuser> ',
'\r\nmal-user> ', '\nmal-user> '],
timeout=args.test_timeout)
#print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
if ret == "*" or p.before == expected:
if ret == "*" or res == expected:
print " -> SUCCESS"
else:
print " -> FAIL (line %d):" % line_num
print " Expected : %s" % repr(expected)
print " Got : %s" % repr(p.before)
print " Got : %s" % repr(res)
fail_cnt += 1
except EOF:
print "Got EOF"
sys.exit(1)
except TIMEOUT:
print "Got TIMEOUT, received: %s" % repr(p.before)
except:
print "Got Exception"
sys.exit(1)
if fail_cnt > 0: