diff options
Diffstat (limited to 'test-chill/testchill/test.py')
| -rw-r--r-- | test-chill/testchill/test.py | 381 |
1 files changed, 381 insertions, 0 deletions
diff --git a/test-chill/testchill/test.py b/test-chill/testchill/test.py new file mode 100644 index 0000000..c38b98a --- /dev/null +++ b/test-chill/testchill/test.py @@ -0,0 +1,381 @@ +from __future__ import print_function +#TODO: test dependencies +#TODO: expected failures +import itertools +import io +import logging +import pprint +import sys +import traceback + +from . import util + + +class TestResult(object): + """ + The base class for all test results. + """ + _pass = 'pass' + _error = 'error' + _fail = 'fail' + _skipped = 'skipped' + + def __init__(self, testcase, status): + self.testcase_name = testcase.name + self.status = status + testcase.setresult(self) + + @staticmethod + def make_pass(result_type, testcase, *args, **kwargs): + """ + Create and return a passing test result of type result_type. + @param result_type A class that extends TestResult + @param testcase The test case that generated the result + @param *args Additional positional arguments to be passed to result_type.__init__ + @param *kwargs Keyword arguments to be passed to result_type.__init__ + """ + return result_type(testcase, TestResult._pass, *args, **kwargs) + + @staticmethod + def make_error(result_type, testcase, *args, **kwargs): + """ + Create and return a errored test result of type result_type. + @param result_type A class that extends TestResult + @param testcase The test case that generated the result + @param *args Additional positional arguments to be passed to result_type.__init__ + @param *kwargs Keyword arguments to be passed to result_type.__init__ + """ + return result_type(testcase, TestResult._error, *args, **kwargs) + + @staticmethod + def make_fail(result_type, testcase, *args, **kwargs): + """ + Create and return a failed test result of type result_type. + @param result_type A class that extends TestResult + @param testcase The test case that generated the result + @param *args Additional positional arguments to be passed to result_type.__init__ + @param *kwargs Keyword arguments to be passed to result_type.__init__ + """ + return result_type(testcase, TestResult._fail, *args, **kwargs) + + @staticmethod + def make_skipped(result_type, testcase, *args, **kwargs): + """ + Create and return a skipped test result of type result_type. + @param result_type A class that extends TestResult + @param testcase The test case that generated the result + @param *args Additional positional arguments to be passed to result_type.__init__ + @param *kwargs Keyword arguments to be passed to result_type.__init__ + """ + return result_type(testcase, TestResult._skipped, *args, **kwargs) + + def passed(self): + """ Return true iff the testcase passed. """ + return self.status == TestResult._pass + + def errored(self): + """ Return true iff the testcase passed. """ + return self.status == TestResult._error + + def failed(self): + """ Return true iff the testcase passed. """ + return self.status == TestResult._fail + + def skipped(self): + """ Return true iff the testcase was skipped """ + return self.status == TestResult._skipped + + def pprint_dict(self): + """ + Return a dict that is ideal for passing to pprint. + """ + return {'testcase_name': self.testcase_name, 'status':self.status} + + def pretty_print(self, width=60, outfile=sys.stdout): + """ + Print result to a file in a human readable way. + """ + print('='*width, end='\n', file=outfile) + print("{}: {}".format(self.status, self.testcase_name), end='\n', file=outfile) + print('-'*width, end='\n', file=outfile) + print(self.pretty_message(), end='\n', file=outfile) + print('-'*width, end='\n', file=outfile) + + def pretty_message(self): + """ Return a message to be printed by pretty_print. Returns an empyt string if not overriden. """ + return '' + + + +class FailedTestResult(TestResult): + """ + A basic implementation of TestResult for failed tests. + """ + def __init__(self, testcase, status=TestResult._fail, reason=None): + super(FailedTestResult, self).__init__(testcase, status) + self.reason = reason + + def pprint_dict(self): + """ + Return a dict that is ideal for passing to pprint. + """ + ppdict = super(FailedTestResult, self).pprint_dict() + ppdict['reason'] = self.reason + return ppdict + + def pretty_message(self): + return self.reason + + +class CompoundTestResult(TestResult): + """ + A TestResult returned by running a sequencial test case + """ + def __init__(self, testcase, results): + super(CompoundTestResult, self).__init__(testcase, None) + self.sub_results = results + status_list = [r.status for r in results] + if TestResult._fail in status_list: + self.status = TestResult._fail + elif TestResult._error in status_list: + self.status = TestResult._error + elif TestResult._pass in status_list: + self.status = TestResult._pass + else: + self.status = TestResult._skipped + + def pprint_dict(self): + """ + Returns a dict that is ideal for passing to pprint. + """ + ppdict = super(CompoundTestResult, self).pprint_dict() + ppdict['sub_results'] = list(s.pprint_dict() for s in self.sub_results) + return ppdict + + def pretty_message(self): + return '\n'.join( + "{}: {}{}".format( + st.status, + st.testcase_name, + '\n' + st.pretty_message() if st.status in [TestResult._fail, TestResult._error] else '') + for st in self.sub_results) + + +class SubTestResult(TestResult): + """ + A TestResult for a subtest in a sequencial test case. + """ + def __init__(self, subtest_name, inner_result): + """ + @param subtest_name The name of the subtest. + @param inner_result The result returned from running the subtest. + """ + super(SubTestResult, self).__init__(inner_result.testcase, inner_result.status) + self.inner_result = inner_result + + def pprint_dict(self): + """ + Return a dict that is ideal for passing to pprint. + """ + ppdict = super(CompoundTestResult, self).pprint_dict() + ppdict['inner_result'] = self.inner_result.pprint_dict() + return ppdict + + +class UnhandledExceptionTestResult(TestResult): + """ + A TestResult returned for exceptions that the test case failed to handle. + """ + def __init__(self, testcase, status, exc_type, exc_value, exc_traceback): + super(UnhandledExceptionTestResult, self).__init__(testcase, status) + self.exception_type = exc_type + self.exception_value = exc_value + if not exc_traceback is None: + sio = util.StringIO() + traceback.print_exception(self.exception_type, self.exception_value, exc_traceback, file=sio) + self.exception_message = sio.getvalue() + else: + self.exception_message = "{}: {}".format(str(exc_type), str(exc_value)) + + def pprint_dict(self): + """ + Return a dict that is ideal for passing to pprint. + """ + ppdict = super(UnhandledExceptionTestResult, self).pprint_dict() + ppdict['exception_type'] = self.exception_type + ppdict['exception_value'] = self.exception_value + ppdict['exception_message'] = self.exception_message + return ppdict + + def pretty_message(self): + return self.exception_message + + +class TestCase(object): + """ + Base class for all test cases + """ + def __init__(self, name=None): + """ + @param name A unique test case name. + """ + self.name = name + + def setUp(self): + """ + Called imediately before a testcase is executed. + """ + pass + + def run(self): + """ + Run the test case, and return its result. + """ + raise NotImplementedError + + def tearDown(self): + """ + Called imediately after a testcase is executed. + """ + pass + + def catch(self, exc): + """ + Called when run raises an exception. If the test case + knows how to handle it, it should return it's own result or None. + Otherwise, return the original exception. + """ + return exc + + def setresult(self, test_result): + """ + Called after a test issues a result and before tearDown is called. + """ + self.test_result = test_result + + def make_pass(self, result_type=TestResult, *args, **kwargs): + """ + Make a passed result for this testcase. + """ + return TestResult.make_pass(result_type, self, *args, **kwargs) + + def make_fail(self, result_type=FailedTestResult, *args, **kwargs): + """ + Make a failed result for this testcase. + """ + return TestResult.make_fail(result_type, self, *args, **kwargs) + + +class SequencialTestCase(TestCase): + """ + A test case that executes a sequence of subtests until + one fails. + """ + def __init__(self, name): + super(SequencialTestCase, self).__init__(name) + self.tests = [] + + def add_subtest(self, subtest_name, subtest_func): + """ + Add a subtest. + """ + self.tests.append((subtest_name, subtest_func)) + + def run(self): + return CompoundTestResult(self, list(self._runall())) + + def _runall(self): + return _rungen([SubTestCase(name, func) for name, func in self.tests], failfast=True) + + +class SubTestCase(TestCase): + """ + A subtest of a sequncial test. + """ + def __init__(self, name, func): + super(SubTestCase, self).__init__(name) + self.run = lambda: func(self) + + +def run(tclist, failfast=False): + """ + Run all test cases in tclist and return a list of thier results. + """ + return list(_rungen(tclist, failfast)) + +def _rungen(tclist, failfast=False): + """ + A generator for running tests internally. + """ + for tc in tclist: + result = None + tc.setUp() + try: + result = _result(tc.run(), tc) + except Exception as ex: + result = _result(tc.catch(ex), tc) + tc.tearDown() + yield result + if failfast and (result.failed() or result.errored()): + break + +def _result(res, tc): + """ + Convert res to a TestResult object. + If res is a TestResult object, give it back. + If res is an Exception, return an UnandledExceptionTestResult. + If res is something else, discard it and return a passed TestResult. + """ + if isinstance(res, TestResult): + return res + elif isinstance(res, Exception): + logging.info('uncaught exception: {}'.format(str(res))) + return TestResult.make_error(UnhandledExceptionTestResult, tc, *(sys.exc_info())) + else: + return TestResult.make_pass(TestResult, tc) + +def pprint_results(result_iter, outfile=sys.stdout): + """ + Print pprint version of test results to a file-like object. + @param result_iter An iterator of results to print. + @param outfile An opened file-like object to print to (defaults to stdout). + """ + status_func = lambda r: r.status + result_iter = sorted(result_iter, key=status_func) + status_dict = dict(iter((k, list(map(lambda tc: tc.pprint_dict(), g))) for k, g in itertools.groupby(result_iter, status_func))) + pprint.pprint(status_dict, stream=outfile) + +def pretty_print_results( + result_iter, + count_by_status=True, exclude_passed=True, exclude_skipped=True, exclude_failed=False, + exclude_errored=False, sort_by_status=True, width=60, outfile=sys.stdout): + """ + Print iterator of TestResults in a human readable format to a file-like object. + @param result_iter An iterator of TestResult objects to print. + @param count_by_status Print the number of tests for each status (defaults to True). + @param exclude_passed Exclude passed test results from printing (defaults to True). + @param exclude_skipped Exclude skipped test results from printing (defaults to True). + @param exclude_failed Exclude failed test results from printing (defaults to False). + @param exclude_errored Exclude errored test results from printing (defaults to False). + @param sort_by_status Print test results in order of status: passed, errored, failed, then skipped (defaults to True). + @param width Printing width (defaults to 60). + @param outfile A file-like object to print to (defaults to stdout). + """ + result_list = list(result_iter) + status_func = lambda r: r.status + if sort_by_status: + #TODO: printing order + result_iter = sorted(result_iter, key=status_func) + + if count_by_status: + print('Passed: {}'.format(len([tr for tr in result_list if tr.passed()])), file=outfile) + print('Errors: {}'.format(len([tr for tr in result_list if tr.errored()])), file=outfile) + print('Failed: {}'.format(len([tr for tr in result_list if tr.failed()])), file=outfile) + print('Skipped: {}'.format(len([tr for tr in result_list if tr.skipped()])), file=outfile) + #TODO: something that doesn't expose TestResult._* + print_status = set(itertools.compress([TestResult._pass, TestResult._error, TestResult._fail, TestResult._skipped], + map(lambda n: not n, [exclude_passed, exclude_errored, exclude_failed, exclude_skipped]))) + for tr in (r for r in result_list if r.status in print_status): + tr.pretty_print(width=width, outfile=outfile) + + |