2019-09-30 20:44:12 +03:00
|
|
|
#!/usr/bin/env python
|
|
|
|
#
|
|
|
|
# Copyright (c) Facebook, Inc. and its affiliates.
|
|
|
|
#
|
|
|
|
"""
|
|
|
|
This file contains the main module code for Python test programs.
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
import contextlib
|
|
|
|
import ctypes
|
|
|
|
import fnmatch
|
|
|
|
import json
|
|
|
|
import logging
|
|
|
|
import optparse
|
|
|
|
import os
|
|
|
|
import platform
|
|
|
|
import re
|
|
|
|
import sys
|
|
|
|
import tempfile
|
|
|
|
import time
|
|
|
|
import traceback
|
|
|
|
import unittest
|
|
|
|
import warnings
|
|
|
|
|
|
|
|
# Hide warning about importing "imp"; remove once python2 is gone.
|
|
|
|
with warnings.catch_warnings():
|
|
|
|
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
|
|
|
import imp
|
|
|
|
|
|
|
|
try:
|
|
|
|
from StringIO import StringIO
|
|
|
|
except ImportError:
|
|
|
|
from io import StringIO
|
|
|
|
try:
|
|
|
|
import coverage
|
|
|
|
except ImportError:
|
|
|
|
coverage = None # type: ignore
|
|
|
|
try:
|
|
|
|
from importlib.machinery import SourceFileLoader
|
|
|
|
except ImportError:
|
|
|
|
SourceFileLoader = None # type: ignore
|
|
|
|
|
|
|
|
|
|
|
|
class get_cpu_instr_counter(object):
|
|
|
|
def read(self):
|
|
|
|
# TODO
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
EXIT_CODE_SUCCESS = 0
|
|
|
|
EXIT_CODE_TEST_FAILURE = 70
|
|
|
|
|
|
|
|
|
|
|
|
class TestStatus(object):
|
|
|
|
|
|
|
|
ABORTED = "FAILURE"
|
|
|
|
PASSED = "SUCCESS"
|
|
|
|
FAILED = "FAILURE"
|
|
|
|
EXPECTED_FAILURE = "SUCCESS"
|
|
|
|
UNEXPECTED_SUCCESS = "FAILURE"
|
|
|
|
SKIPPED = "ASSUMPTION_VIOLATION"
|
|
|
|
|
|
|
|
|
|
|
|
class PathMatcher(object):
|
|
|
|
def __init__(self, include_patterns, omit_patterns):
|
|
|
|
self.include_patterns = include_patterns
|
|
|
|
self.omit_patterns = omit_patterns
|
|
|
|
|
|
|
|
def omit(self, path):
|
|
|
|
"""
|
|
|
|
Omit iff matches any of the omit_patterns or the include patterns are
|
|
|
|
not empty and none is matched
|
|
|
|
"""
|
|
|
|
path = os.path.realpath(path)
|
|
|
|
return any(fnmatch.fnmatch(path, p) for p in self.omit_patterns) or (
|
|
|
|
self.include_patterns
|
|
|
|
and not any(fnmatch.fnmatch(path, p) for p in self.include_patterns)
|
|
|
|
)
|
|
|
|
|
|
|
|
def include(self, path):
|
|
|
|
return not self.omit(path)
|
|
|
|
|
|
|
|
|
|
|
|
class DebugWipeFinder(object):
|
|
|
|
"""
|
|
|
|
PEP 302 finder that uses a DebugWipeLoader for all files which do not need
|
|
|
|
coverage
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self, matcher):
|
|
|
|
self.matcher = matcher
|
|
|
|
|
|
|
|
def find_module(self, fullname, path=None):
|
|
|
|
_, _, basename = fullname.rpartition(".")
|
|
|
|
try:
|
|
|
|
fd, pypath, (_, _, kind) = imp.find_module(basename, path)
|
|
|
|
except Exception:
|
Fix bug in optimizing module loader for coverage collection
Summary:
In coverage collection mode a special module loader is prepended to
`sys.meta_path`. In very specific conditions this module loader can end up
returning a loader pointing to a _completely wrong module_. When importing
symbols from the wrong module errors occur.
The conditions to trigger the bug are:
- running in coverage collection mode, enabling the custom loader
- the test binary is a zip (e.g. par_style=fastzip)
- having a module name where the end part matches the name of a builtin Python
module
When these conditions were met, the special loader would return the builtin
Python module instead of the expected module. E.g. when loading a module like
`myapp.somemod.platform` in a zip style binary.
The custom loader first calls `imp.find_module()` to find the module it wants
to return a wrapped loader for. This fails for modules included in the test
binary, because the builtin `imp` module can not load from zips. This was the
trigger leading to the call to the buggy code.
When the initial call to `imp.find_module()` failed, the custom loader would
try a second call, asking the internal loader to this time try any path on
`sys.path`. For most module names this call would also fail, making the custom
loader return `None`, after which Python tries other loaders on `sys.path`.
However, when the final part of the module that was asked to load matches the
name of a Python builtin module, then the second call to the `imp` module would
succeed, returning a loader for the builtin module. E.g. `platform` when asking
for `myapp.somemod.platform`.
This diff fixes the issue by removing the broken second call to the internal
loader. This will never have worked, we just have not triggered or noticed
triggering the wrong loading before.
Differential Revision: D20798119
fbshipit-source-id: dffb54e308106a81af21b63c5ee64c6ca2041920
2020-04-02 12:40:00 +03:00
|
|
|
# Finding without hooks using the imp module failed. One reason
|
|
|
|
# could be that there is a zip file on sys.path. The imp module
|
|
|
|
# does not support loading from there. Leave finding this module to
|
|
|
|
# the others finders in sys.meta_path.
|
|
|
|
return None
|
2019-09-30 20:44:12 +03:00
|
|
|
|
|
|
|
if hasattr(fd, "close"):
|
|
|
|
fd.close()
|
|
|
|
if kind != imp.PY_SOURCE:
|
|
|
|
return None
|
|
|
|
if self.matcher.include(pypath):
|
|
|
|
return None
|
|
|
|
|
|
|
|
"""
|
|
|
|
This is defined to match CPython's PyVarObject struct
|
|
|
|
"""
|
|
|
|
|
|
|
|
class PyVarObject(ctypes.Structure):
|
|
|
|
_fields_ = [
|
|
|
|
("ob_refcnt", ctypes.c_long),
|
|
|
|
("ob_type", ctypes.c_void_p),
|
|
|
|
("ob_size", ctypes.c_ulong),
|
|
|
|
]
|
|
|
|
|
|
|
|
class DebugWipeLoader(SourceFileLoader):
|
|
|
|
"""
|
|
|
|
PEP302 loader that zeros out debug information before execution
|
|
|
|
"""
|
|
|
|
|
|
|
|
def get_code(self, fullname):
|
|
|
|
code = super(DebugWipeLoader, self).get_code(fullname)
|
|
|
|
if code:
|
|
|
|
# Ideally we'd do
|
|
|
|
# code.co_lnotab = b''
|
|
|
|
# But code objects are READONLY. Not to worry though; we'll
|
|
|
|
# directly modify CPython's object
|
|
|
|
code_impl = PyVarObject.from_address(id(code.co_lnotab))
|
|
|
|
code_impl.ob_size = 0
|
|
|
|
return code
|
|
|
|
|
|
|
|
return DebugWipeLoader(fullname, pypath)
|
|
|
|
|
|
|
|
|
|
|
|
def optimize_for_coverage(cov, include_patterns, omit_patterns):
|
|
|
|
"""
|
|
|
|
We get better performance if we zero out debug information for files which
|
|
|
|
we're not interested in. Only available in CPython 3.3+
|
|
|
|
"""
|
|
|
|
matcher = PathMatcher(include_patterns, omit_patterns)
|
|
|
|
if SourceFileLoader and platform.python_implementation() == "CPython":
|
|
|
|
sys.meta_path.insert(0, DebugWipeFinder(matcher))
|
|
|
|
|
|
|
|
|
|
|
|
class TeeStream(object):
|
|
|
|
def __init__(self, *streams):
|
|
|
|
self._streams = streams
|
|
|
|
|
|
|
|
def write(self, data):
|
|
|
|
for stream in self._streams:
|
|
|
|
stream.write(data)
|
|
|
|
|
|
|
|
def flush(self):
|
|
|
|
for stream in self._streams:
|
|
|
|
stream.flush()
|
|
|
|
|
|
|
|
def isatty(self):
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
class CallbackStream(object):
|
|
|
|
def __init__(self, callback, bytes_callback=None, orig=None):
|
|
|
|
self._callback = callback
|
|
|
|
self._fileno = orig.fileno() if orig else None
|
|
|
|
|
|
|
|
# Python 3 APIs:
|
|
|
|
# - `encoding` is a string holding the encoding name
|
|
|
|
# - `errors` is a string holding the error-handling mode for encoding
|
|
|
|
# - `buffer` should look like an io.BufferedIOBase object
|
|
|
|
|
|
|
|
self.errors = orig.errors if orig else None
|
|
|
|
if bytes_callback:
|
|
|
|
# those members are only on the io.TextIOWrapper
|
|
|
|
self.encoding = orig.encoding if orig else "UTF-8"
|
|
|
|
self.buffer = CallbackStream(bytes_callback, orig=orig)
|
|
|
|
|
|
|
|
def write(self, data):
|
|
|
|
self._callback(data)
|
|
|
|
|
|
|
|
def flush(self):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def isatty(self):
|
|
|
|
return False
|
|
|
|
|
|
|
|
def fileno(self):
|
|
|
|
return self._fileno
|
|
|
|
|
|
|
|
|
|
|
|
class BuckTestResult(unittest._TextTestResult):
|
|
|
|
"""
|
|
|
|
Our own TestResult class that outputs data in a format that can be easily
|
|
|
|
parsed by buck's test runner.
|
|
|
|
"""
|
|
|
|
|
|
|
|
_instr_counter = get_cpu_instr_counter()
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
self, stream, descriptions, verbosity, show_output, main_program, suite
|
|
|
|
):
|
|
|
|
super(BuckTestResult, self).__init__(stream, descriptions, verbosity)
|
|
|
|
self._main_program = main_program
|
|
|
|
self._suite = suite
|
|
|
|
self._results = []
|
|
|
|
self._current_test = None
|
|
|
|
self._saved_stdout = sys.stdout
|
|
|
|
self._saved_stderr = sys.stderr
|
|
|
|
self._show_output = show_output
|
|
|
|
|
|
|
|
def getResults(self):
|
|
|
|
return self._results
|
|
|
|
|
|
|
|
def startTest(self, test):
|
|
|
|
super(BuckTestResult, self).startTest(test)
|
|
|
|
|
|
|
|
# Pass in the real stdout and stderr filenos. We can't really do much
|
|
|
|
# here to intercept callers who directly operate on these fileno
|
|
|
|
# objects.
|
|
|
|
sys.stdout = CallbackStream(
|
|
|
|
self.addStdout, self.addStdoutBytes, orig=sys.stdout
|
|
|
|
)
|
|
|
|
sys.stderr = CallbackStream(
|
|
|
|
self.addStderr, self.addStderrBytes, orig=sys.stderr
|
|
|
|
)
|
|
|
|
self._current_test = test
|
|
|
|
self._test_start_time = time.time()
|
|
|
|
self._current_status = TestStatus.ABORTED
|
|
|
|
self._messages = []
|
|
|
|
self._stacktrace = None
|
|
|
|
self._stdout = ""
|
|
|
|
self._stderr = ""
|
|
|
|
self._start_instr_count = self._instr_counter.read()
|
|
|
|
|
|
|
|
def _find_next_test(self, suite):
|
|
|
|
"""
|
|
|
|
Find the next test that has not been run.
|
|
|
|
"""
|
|
|
|
|
|
|
|
for test in suite:
|
|
|
|
|
|
|
|
# We identify test suites by test that are iterable (as is done in
|
|
|
|
# the builtin python test harness). If we see one, recurse on it.
|
|
|
|
if hasattr(test, "__iter__"):
|
|
|
|
test = self._find_next_test(test)
|
|
|
|
|
|
|
|
# The builtin python test harness sets test references to `None`
|
|
|
|
# after they have run, so we know we've found the next test up
|
|
|
|
# if it's not `None`.
|
|
|
|
if test is not None:
|
|
|
|
return test
|
|
|
|
|
|
|
|
def stopTest(self, test):
|
|
|
|
sys.stdout = self._saved_stdout
|
|
|
|
sys.stderr = self._saved_stderr
|
|
|
|
|
|
|
|
super(BuckTestResult, self).stopTest(test)
|
|
|
|
|
|
|
|
# If a failure occured during module/class setup, then this "test" may
|
|
|
|
# actually be a `_ErrorHolder`, which doesn't contain explicit info
|
|
|
|
# about the upcoming test. Since we really only care about the test
|
|
|
|
# name field (i.e. `_testMethodName`), we use that to detect an actual
|
|
|
|
# test cases, and fall back to looking the test up from the suite
|
|
|
|
# otherwise.
|
|
|
|
if not hasattr(test, "_testMethodName"):
|
|
|
|
test = self._find_next_test(self._suite)
|
|
|
|
|
|
|
|
result = {
|
|
|
|
"testCaseName": "{0}.{1}".format(
|
|
|
|
test.__class__.__module__, test.__class__.__name__
|
|
|
|
),
|
|
|
|
"testCase": test._testMethodName,
|
|
|
|
"type": self._current_status,
|
|
|
|
"time": int((time.time() - self._test_start_time) * 1000),
|
|
|
|
"message": os.linesep.join(self._messages),
|
|
|
|
"stacktrace": self._stacktrace,
|
|
|
|
"stdOut": self._stdout,
|
|
|
|
"stdErr": self._stderr,
|
|
|
|
}
|
|
|
|
|
|
|
|
# TestPilot supports an instruction count field.
|
|
|
|
if "TEST_PILOT" in os.environ:
|
|
|
|
result["instrCount"] = (
|
|
|
|
int(self._instr_counter.read() - self._start_instr_count),
|
|
|
|
)
|
|
|
|
|
|
|
|
self._results.append(result)
|
|
|
|
self._current_test = None
|
|
|
|
|
|
|
|
def stopTestRun(self):
|
|
|
|
cov = self._main_program.get_coverage()
|
|
|
|
if cov is not None:
|
|
|
|
self._results.append({"coverage": cov})
|
|
|
|
|
|
|
|
@contextlib.contextmanager
|
|
|
|
def _withTest(self, test):
|
|
|
|
self.startTest(test)
|
|
|
|
yield
|
|
|
|
self.stopTest(test)
|
|
|
|
|
|
|
|
def _setStatus(self, test, status, message=None, stacktrace=None):
|
|
|
|
assert test == self._current_test
|
|
|
|
self._current_status = status
|
|
|
|
self._stacktrace = stacktrace
|
|
|
|
if message is not None:
|
|
|
|
if message.endswith(os.linesep):
|
|
|
|
message = message[:-1]
|
|
|
|
self._messages.append(message)
|
|
|
|
|
|
|
|
def setStatus(self, test, status, message=None, stacktrace=None):
|
|
|
|
# addError() may be called outside of a test if one of the shared
|
|
|
|
# fixtures (setUpClass/tearDownClass/setUpModule/tearDownModule)
|
|
|
|
# throws an error.
|
|
|
|
#
|
|
|
|
# In this case, create a fake test result to record the error.
|
|
|
|
if self._current_test is None:
|
|
|
|
with self._withTest(test):
|
|
|
|
self._setStatus(test, status, message, stacktrace)
|
|
|
|
else:
|
|
|
|
self._setStatus(test, status, message, stacktrace)
|
|
|
|
|
|
|
|
def setException(self, test, status, excinfo):
|
|
|
|
exctype, value, tb = excinfo
|
|
|
|
self.setStatus(
|
|
|
|
test,
|
|
|
|
status,
|
|
|
|
"{0}: {1}".format(exctype.__name__, value),
|
|
|
|
"".join(traceback.format_tb(tb)),
|
|
|
|
)
|
|
|
|
|
|
|
|
def addSuccess(self, test):
|
|
|
|
super(BuckTestResult, self).addSuccess(test)
|
|
|
|
self.setStatus(test, TestStatus.PASSED)
|
|
|
|
|
|
|
|
def addError(self, test, err):
|
|
|
|
super(BuckTestResult, self).addError(test, err)
|
|
|
|
self.setException(test, TestStatus.ABORTED, err)
|
|
|
|
|
|
|
|
def addFailure(self, test, err):
|
|
|
|
super(BuckTestResult, self).addFailure(test, err)
|
|
|
|
self.setException(test, TestStatus.FAILED, err)
|
|
|
|
|
|
|
|
def addSkip(self, test, reason):
|
|
|
|
super(BuckTestResult, self).addSkip(test, reason)
|
|
|
|
self.setStatus(test, TestStatus.SKIPPED, "Skipped: %s" % (reason,))
|
|
|
|
|
|
|
|
def addExpectedFailure(self, test, err):
|
|
|
|
super(BuckTestResult, self).addExpectedFailure(test, err)
|
|
|
|
self.setException(test, TestStatus.EXPECTED_FAILURE, err)
|
|
|
|
|
|
|
|
def addUnexpectedSuccess(self, test):
|
|
|
|
super(BuckTestResult, self).addUnexpectedSuccess(test)
|
|
|
|
self.setStatus(test, TestStatus.UNEXPECTED_SUCCESS, "Unexpected success")
|
|
|
|
|
|
|
|
def addStdout(self, val):
|
|
|
|
self._stdout += val
|
|
|
|
if self._show_output:
|
|
|
|
self._saved_stdout.write(val)
|
|
|
|
self._saved_stdout.flush()
|
|
|
|
|
|
|
|
def addStdoutBytes(self, val):
|
|
|
|
string = val.decode("utf-8", errors="backslashreplace")
|
|
|
|
self.addStdout(string)
|
|
|
|
|
|
|
|
def addStderr(self, val):
|
|
|
|
self._stderr += val
|
|
|
|
if self._show_output:
|
|
|
|
self._saved_stderr.write(val)
|
|
|
|
self._saved_stderr.flush()
|
|
|
|
|
|
|
|
def addStderrBytes(self, val):
|
|
|
|
string = val.decode("utf-8", errors="backslashreplace")
|
|
|
|
self.addStderr(string)
|
|
|
|
|
|
|
|
|
|
|
|
class BuckTestRunner(unittest.TextTestRunner):
|
|
|
|
def __init__(self, main_program, suite, show_output=True, **kwargs):
|
|
|
|
super(BuckTestRunner, self).__init__(**kwargs)
|
|
|
|
self.show_output = show_output
|
|
|
|
self._main_program = main_program
|
|
|
|
self._suite = suite
|
|
|
|
|
|
|
|
def _makeResult(self):
|
|
|
|
return BuckTestResult(
|
|
|
|
self.stream,
|
|
|
|
self.descriptions,
|
|
|
|
self.verbosity,
|
|
|
|
self.show_output,
|
|
|
|
self._main_program,
|
|
|
|
self._suite,
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def _format_test_name(test_class, attrname):
|
|
|
|
return "{0}.{1}.{2}".format(test_class.__module__, test_class.__name__, attrname)
|
|
|
|
|
|
|
|
|
|
|
|
class StderrLogHandler(logging.StreamHandler):
|
|
|
|
"""
|
|
|
|
This class is very similar to logging.StreamHandler, except that it
|
|
|
|
always uses the current sys.stderr object.
|
|
|
|
|
|
|
|
StreamHandler caches the current sys.stderr object when it is constructed.
|
|
|
|
This makes it behave poorly in unit tests, which may replace sys.stderr
|
|
|
|
with a StringIO buffer during tests. The StreamHandler will continue using
|
|
|
|
the old sys.stderr object instead of the desired StringIO buffer.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
logging.Handler.__init__(self)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def stream(self):
|
|
|
|
return sys.stderr
|
|
|
|
|
|
|
|
|
|
|
|
class RegexTestLoader(unittest.TestLoader):
|
|
|
|
def __init__(self, regex=None):
|
|
|
|
self.regex = regex
|
|
|
|
super(RegexTestLoader, self).__init__()
|
|
|
|
|
|
|
|
def getTestCaseNames(self, testCaseClass):
|
|
|
|
"""
|
|
|
|
Return a sorted sequence of method names found within testCaseClass
|
|
|
|
"""
|
|
|
|
|
|
|
|
testFnNames = super(RegexTestLoader, self).getTestCaseNames(testCaseClass)
|
|
|
|
if self.regex is None:
|
|
|
|
return testFnNames
|
|
|
|
robj = re.compile(self.regex)
|
|
|
|
matched = []
|
|
|
|
for attrname in testFnNames:
|
|
|
|
fullname = _format_test_name(testCaseClass, attrname)
|
|
|
|
if robj.search(fullname):
|
|
|
|
matched.append(attrname)
|
|
|
|
return matched
|
|
|
|
|
|
|
|
|
|
|
|
class Loader(object):
|
|
|
|
|
|
|
|
suiteClass = unittest.TestSuite
|
|
|
|
|
|
|
|
def __init__(self, modules, regex=None):
|
|
|
|
self.modules = modules
|
|
|
|
self.regex = regex
|
|
|
|
|
|
|
|
def load_all(self):
|
|
|
|
loader = RegexTestLoader(self.regex)
|
|
|
|
test_suite = self.suiteClass()
|
|
|
|
for module_name in self.modules:
|
|
|
|
__import__(module_name, level=0)
|
|
|
|
module = sys.modules[module_name]
|
|
|
|
module_suite = loader.loadTestsFromModule(module)
|
|
|
|
test_suite.addTest(module_suite)
|
|
|
|
return test_suite
|
|
|
|
|
|
|
|
def load_args(self, args):
|
|
|
|
loader = RegexTestLoader(self.regex)
|
|
|
|
|
|
|
|
suites = []
|
|
|
|
for arg in args:
|
|
|
|
suite = loader.loadTestsFromName(arg)
|
|
|
|
# loadTestsFromName() can only process names that refer to
|
|
|
|
# individual test functions or modules. It can't process package
|
|
|
|
# names. If there were no module/function matches, check to see if
|
|
|
|
# this looks like a package name.
|
|
|
|
if suite.countTestCases() != 0:
|
|
|
|
suites.append(suite)
|
|
|
|
continue
|
|
|
|
|
|
|
|
# Load all modules whose name is <arg>.<something>
|
|
|
|
prefix = arg + "."
|
|
|
|
for module in self.modules:
|
|
|
|
if module.startswith(prefix):
|
|
|
|
suite = loader.loadTestsFromName(module)
|
|
|
|
suites.append(suite)
|
|
|
|
|
|
|
|
return loader.suiteClass(suites)
|
|
|
|
|
|
|
|
|
2021-07-09 16:17:52 +03:00
|
|
|
_COVERAGE_INI = """\
|
2019-09-30 20:44:12 +03:00
|
|
|
[report]
|
|
|
|
exclude_lines =
|
|
|
|
pragma: no cover
|
|
|
|
pragma: nocover
|
|
|
|
pragma:.*no${PLATFORM}
|
|
|
|
pragma:.*no${PY_IMPL}${PY_MAJOR}${PY_MINOR}
|
|
|
|
pragma:.*no${PY_IMPL}${PY_MAJOR}
|
|
|
|
pragma:.*nopy${PY_MAJOR}
|
|
|
|
pragma:.*nopy${PY_MAJOR}${PY_MINOR}
|
2021-07-09 16:17:52 +03:00
|
|
|
"""
|
2019-09-30 20:44:12 +03:00
|
|
|
|
|
|
|
|
|
|
|
class MainProgram(object):
|
|
|
|
"""
|
|
|
|
This class implements the main program. It can be subclassed by
|
|
|
|
users who wish to customize some parts of the main program.
|
|
|
|
(Adding additional command line options, customizing test loading, etc.)
|
|
|
|
"""
|
|
|
|
|
|
|
|
DEFAULT_VERBOSITY = 2
|
|
|
|
|
|
|
|
def __init__(self, argv):
|
|
|
|
self.init_option_parser()
|
|
|
|
self.parse_options(argv)
|
|
|
|
self.setup_logging()
|
|
|
|
|
|
|
|
def init_option_parser(self):
|
|
|
|
usage = "%prog [options] [TEST] ..."
|
|
|
|
op = optparse.OptionParser(usage=usage, add_help_option=False)
|
|
|
|
self.option_parser = op
|
|
|
|
|
|
|
|
op.add_option(
|
|
|
|
"--hide-output",
|
|
|
|
dest="show_output",
|
|
|
|
action="store_false",
|
|
|
|
default=True,
|
|
|
|
help="Suppress data that tests print to stdout/stderr, and only "
|
|
|
|
"show it if the test fails.",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"-o",
|
|
|
|
"--output",
|
|
|
|
help="Write results to a file in a JSON format to be read by Buck",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"-f",
|
|
|
|
"--failfast",
|
|
|
|
action="store_true",
|
|
|
|
default=False,
|
|
|
|
help="Stop after the first failure",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"-l",
|
|
|
|
"--list-tests",
|
|
|
|
action="store_true",
|
|
|
|
dest="list",
|
|
|
|
default=False,
|
|
|
|
help="List tests and exit",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"-r",
|
|
|
|
"--regex",
|
|
|
|
default=None,
|
|
|
|
help="Regex to apply to tests, to only run those tests",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"--collect-coverage",
|
|
|
|
action="store_true",
|
|
|
|
default=False,
|
|
|
|
help="Collect test coverage information",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"--coverage-include",
|
|
|
|
default="*",
|
|
|
|
help='File globs to include in converage (split by ",")',
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"--coverage-omit",
|
|
|
|
default="",
|
|
|
|
help='File globs to omit from converage (split by ",")',
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"--logger",
|
|
|
|
action="append",
|
|
|
|
metavar="<category>=<level>",
|
|
|
|
default=[],
|
|
|
|
help="Configure log levels for specific logger categories",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"-q",
|
|
|
|
"--quiet",
|
|
|
|
action="count",
|
|
|
|
default=0,
|
|
|
|
help="Decrease the verbosity (may be specified multiple times)",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"-v",
|
|
|
|
"--verbosity",
|
|
|
|
action="count",
|
|
|
|
default=self.DEFAULT_VERBOSITY,
|
|
|
|
help="Increase the verbosity (may be specified multiple times)",
|
|
|
|
)
|
|
|
|
op.add_option(
|
|
|
|
"-?", "--help", action="help", help="Show this help message and exit"
|
|
|
|
)
|
|
|
|
|
|
|
|
def parse_options(self, argv):
|
|
|
|
self.options, self.test_args = self.option_parser.parse_args(argv[1:])
|
|
|
|
self.options.verbosity -= self.options.quiet
|
|
|
|
|
|
|
|
if self.options.collect_coverage and coverage is None:
|
|
|
|
self.option_parser.error("coverage module is not available")
|
|
|
|
self.options.coverage_include = self.options.coverage_include.split(",")
|
|
|
|
if self.options.coverage_omit == "":
|
|
|
|
self.options.coverage_omit = []
|
|
|
|
else:
|
|
|
|
self.options.coverage_omit = self.options.coverage_omit.split(",")
|
|
|
|
|
|
|
|
def setup_logging(self):
|
|
|
|
# Configure the root logger to log at INFO level.
|
|
|
|
# This is similar to logging.basicConfig(), but uses our
|
|
|
|
# StderrLogHandler instead of a StreamHandler.
|
|
|
|
fmt = logging.Formatter("%(pathname)s:%(lineno)s: %(message)s")
|
|
|
|
log_handler = StderrLogHandler()
|
|
|
|
log_handler.setFormatter(fmt)
|
|
|
|
root_logger = logging.getLogger()
|
|
|
|
root_logger.addHandler(log_handler)
|
|
|
|
root_logger.setLevel(logging.INFO)
|
|
|
|
|
|
|
|
level_names = {
|
|
|
|
"debug": logging.DEBUG,
|
|
|
|
"info": logging.INFO,
|
|
|
|
"warn": logging.WARNING,
|
|
|
|
"warning": logging.WARNING,
|
|
|
|
"error": logging.ERROR,
|
|
|
|
"critical": logging.CRITICAL,
|
|
|
|
"fatal": logging.FATAL,
|
|
|
|
}
|
|
|
|
|
|
|
|
for value in self.options.logger:
|
|
|
|
parts = value.rsplit("=", 1)
|
|
|
|
if len(parts) != 2:
|
|
|
|
self.option_parser.error(
|
|
|
|
"--logger argument must be of the "
|
|
|
|
"form <name>=<level>: %s" % value
|
|
|
|
)
|
|
|
|
name = parts[0]
|
|
|
|
level_name = parts[1].lower()
|
|
|
|
level = level_names.get(level_name)
|
|
|
|
if level is None:
|
|
|
|
self.option_parser.error(
|
|
|
|
"invalid log level %r for log " "category %s" % (parts[1], name)
|
|
|
|
)
|
|
|
|
logging.getLogger(name).setLevel(level)
|
|
|
|
|
|
|
|
def create_loader(self):
|
|
|
|
import __test_modules__
|
|
|
|
|
|
|
|
return Loader(__test_modules__.TEST_MODULES, self.options.regex)
|
|
|
|
|
|
|
|
def load_tests(self):
|
|
|
|
loader = self.create_loader()
|
|
|
|
if self.options.collect_coverage:
|
|
|
|
self.start_coverage()
|
|
|
|
include = self.options.coverage_include
|
|
|
|
omit = self.options.coverage_omit
|
|
|
|
if include and "*" not in include:
|
|
|
|
optimize_for_coverage(self.cov, include, omit)
|
|
|
|
|
|
|
|
if self.test_args:
|
|
|
|
suite = loader.load_args(self.test_args)
|
|
|
|
else:
|
|
|
|
suite = loader.load_all()
|
|
|
|
if self.options.collect_coverage:
|
|
|
|
self.cov.start()
|
|
|
|
return suite
|
|
|
|
|
|
|
|
def get_tests(self, test_suite):
|
|
|
|
tests = []
|
|
|
|
|
|
|
|
for test in test_suite:
|
|
|
|
if isinstance(test, unittest.TestSuite):
|
|
|
|
tests.extend(self.get_tests(test))
|
|
|
|
else:
|
|
|
|
tests.append(test)
|
|
|
|
|
|
|
|
return tests
|
|
|
|
|
|
|
|
def run(self):
|
|
|
|
test_suite = self.load_tests()
|
|
|
|
|
|
|
|
if self.options.list:
|
|
|
|
for test in self.get_tests(test_suite):
|
|
|
|
method_name = getattr(test, "_testMethodName", "")
|
|
|
|
name = _format_test_name(test.__class__, method_name)
|
|
|
|
print(name)
|
|
|
|
return EXIT_CODE_SUCCESS
|
|
|
|
else:
|
|
|
|
result = self.run_tests(test_suite)
|
|
|
|
if self.options.output is not None:
|
|
|
|
with open(self.options.output, "w") as f:
|
|
|
|
json.dump(result.getResults(), f, indent=4, sort_keys=True)
|
|
|
|
if not result.wasSuccessful():
|
|
|
|
return EXIT_CODE_TEST_FAILURE
|
|
|
|
return EXIT_CODE_SUCCESS
|
|
|
|
|
|
|
|
def run_tests(self, test_suite):
|
|
|
|
# Install a signal handler to catch Ctrl-C and display the results
|
|
|
|
# (but only if running >2.6).
|
|
|
|
if sys.version_info[0] > 2 or sys.version_info[1] > 6:
|
|
|
|
unittest.installHandler()
|
|
|
|
|
|
|
|
# Run the tests
|
|
|
|
runner = BuckTestRunner(
|
|
|
|
self,
|
|
|
|
test_suite,
|
|
|
|
verbosity=self.options.verbosity,
|
|
|
|
show_output=self.options.show_output,
|
|
|
|
)
|
|
|
|
result = runner.run(test_suite)
|
|
|
|
|
|
|
|
if self.options.collect_coverage and self.options.show_output:
|
|
|
|
self.cov.stop()
|
|
|
|
try:
|
|
|
|
self.cov.report(file=sys.stdout)
|
|
|
|
except coverage.misc.CoverageException:
|
|
|
|
print("No lines were covered, potentially restricted by file filters")
|
|
|
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
def get_abbr_impl(self):
|
|
|
|
"""Return abbreviated implementation name."""
|
|
|
|
impl = platform.python_implementation()
|
|
|
|
if impl == "PyPy":
|
|
|
|
return "pp"
|
|
|
|
elif impl == "Jython":
|
|
|
|
return "jy"
|
|
|
|
elif impl == "IronPython":
|
|
|
|
return "ip"
|
|
|
|
elif impl == "CPython":
|
|
|
|
return "cp"
|
|
|
|
else:
|
|
|
|
raise RuntimeError("unknown python runtime")
|
|
|
|
|
|
|
|
def start_coverage(self):
|
|
|
|
if not self.options.collect_coverage:
|
|
|
|
return
|
|
|
|
|
2021-07-09 16:17:52 +03:00
|
|
|
with tempfile.NamedTemporaryFile("w", delete=False) as coverage_ini:
|
2019-09-30 20:44:12 +03:00
|
|
|
coverage_ini.write(_COVERAGE_INI)
|
|
|
|
self._coverage_ini_path = coverage_ini.name
|
|
|
|
|
|
|
|
# Keep the original working dir in case tests use os.chdir
|
|
|
|
self._original_working_dir = os.getcwd()
|
|
|
|
|
|
|
|
# for coverage config ignores by platform/python version
|
|
|
|
os.environ["PLATFORM"] = sys.platform
|
|
|
|
os.environ["PY_IMPL"] = self.get_abbr_impl()
|
|
|
|
os.environ["PY_MAJOR"] = str(sys.version_info.major)
|
|
|
|
os.environ["PY_MINOR"] = str(sys.version_info.minor)
|
|
|
|
|
|
|
|
self.cov = coverage.Coverage(
|
|
|
|
include=self.options.coverage_include,
|
|
|
|
omit=self.options.coverage_omit,
|
|
|
|
config_file=coverage_ini.name,
|
|
|
|
)
|
|
|
|
self.cov.erase()
|
|
|
|
self.cov.start()
|
|
|
|
|
|
|
|
def get_coverage(self):
|
|
|
|
if not self.options.collect_coverage:
|
|
|
|
return None
|
|
|
|
|
|
|
|
try:
|
|
|
|
os.remove(self._coverage_ini_path)
|
|
|
|
except OSError:
|
|
|
|
pass # Better to litter than to fail the test
|
|
|
|
|
|
|
|
# Switch back to the original working directory.
|
|
|
|
os.chdir(self._original_working_dir)
|
|
|
|
|
|
|
|
result = {}
|
|
|
|
|
|
|
|
self.cov.stop()
|
|
|
|
|
|
|
|
try:
|
|
|
|
f = StringIO()
|
|
|
|
self.cov.report(file=f)
|
|
|
|
lines = f.getvalue().split("\n")
|
|
|
|
except coverage.misc.CoverageException:
|
|
|
|
# Nothing was covered. That's fine by us
|
|
|
|
return result
|
|
|
|
|
|
|
|
# N.B.: the format of the coverage library's output differs
|
|
|
|
# depending on whether one or more files are in the results
|
|
|
|
for line in lines[2:]:
|
|
|
|
if line.strip("-") == "":
|
|
|
|
break
|
|
|
|
r = line.split()[0]
|
|
|
|
analysis = self.cov.analysis2(r)
|
|
|
|
covString = self.convert_to_diff_cov_str(analysis)
|
|
|
|
if covString:
|
|
|
|
result[r] = covString
|
|
|
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
def convert_to_diff_cov_str(self, analysis):
|
|
|
|
# Info on the format of analysis:
|
|
|
|
# http://nedbatchelder.com/code/coverage/api.html
|
|
|
|
if not analysis:
|
|
|
|
return None
|
|
|
|
numLines = max(
|
|
|
|
analysis[1][-1] if len(analysis[1]) else 0,
|
|
|
|
analysis[2][-1] if len(analysis[2]) else 0,
|
|
|
|
analysis[3][-1] if len(analysis[3]) else 0,
|
|
|
|
)
|
|
|
|
lines = ["N"] * numLines
|
|
|
|
for l in analysis[1]:
|
|
|
|
lines[l - 1] = "C"
|
|
|
|
for l in analysis[2]:
|
|
|
|
lines[l - 1] = "X"
|
|
|
|
for l in analysis[3]:
|
|
|
|
lines[l - 1] = "U"
|
|
|
|
return "".join(lines)
|
|
|
|
|
|
|
|
|
|
|
|
def main(argv):
|
|
|
|
return MainProgram(sys.argv).run()
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
sys.exit(main(sys.argv))
|