path: root/test-chill/testchill/test.py

from __future__ import print_function
#TODO: test dependencies
#TODO: expected failures
import itertools
import io
import logging
import pprint
import sys
import traceback

from . import util


class TestResult(object):
    """
    The base class for all test results.
    """
    _pass = 'pass'
    _error = 'error'
    _fail = 'fail'
    _skipped = 'skipped'
    
    def __init__(self, testcase, status):
        self.testcase_name = testcase.name
        self.status = status
        testcase.setresult(self)
    
    @staticmethod
    def make_pass(result_type, testcase, *args, **kwargs):
        """
        Create and return a passing test result of type result_type.
        @param result_type A class that extends TestResult
        @param testcase The test case that generated the result
        @param *args Additional positional arguments to be passed to result_type.__init__
        @param *kwargs Keyword arguments to be passed to result_type.__init__
        """
        return result_type(testcase, TestResult._pass, *args, **kwargs)
    
    @staticmethod
    def make_error(result_type, testcase, *args, **kwargs):
        """
        Create and return a errored test result of type result_type.
        @param result_type A class that extends TestResult
        @param testcase The test case that generated the result
        @param *args Additional positional arguments to be passed to result_type.__init__
        @param *kwargs Keyword arguments to be passed to result_type.__init__
        """
        return result_type(testcase, TestResult._error, *args, **kwargs)
    
    @staticmethod
    def make_fail(result_type, testcase, *args, **kwargs):
        """
        Create and return a failed test result of type result_type.
        @param result_type A class that extends TestResult
        @param testcase The test case that generated the result
        @param *args Additional positional arguments to be passed to result_type.__init__
        @param *kwargs Keyword arguments to be passed to result_type.__init__
        """
        return result_type(testcase, TestResult._fail, *args, **kwargs)
    
    @staticmethod
    def make_skipped(result_type, testcase, *args, **kwargs):
        """
        Create and return a skipped test result of type result_type.
        @param result_type A class that extends TestResult
        @param testcase The test case that generated the result
        @param *args Additional positional arguments to be passed to result_type.__init__
        @param *kwargs Keyword arguments to be passed to result_type.__init__
        """
        return result_type(testcase, TestResult._skipped, *args, **kwargs)
    
    def passed(self):
        """ Return true iff the testcase passed. """
        return self.status == TestResult._pass
    
    def errored(self):
        """ Return true iff the testcase passed. """
        return self.status == TestResult._error

    def failed(self):
        """ Return true iff the testcase passed. """
        return self.status == TestResult._fail
    
    def skipped(self):
        """ Return true iff the testcase was skipped """
        return self.status == TestResult._skipped
        
    def pprint_dict(self):
        """
        Return a dict that is ideal for passing to pprint.
        """
        return {'testcase_name': self.testcase_name, 'status':self.status}
    
    def pretty_print(self, width=60, outfile=sys.stdout):
        """
        Print result to a file in a human readable way.
        """
        print('='*width, end='\n', file=outfile)
        print("{}: {}".format(self.status, self.testcase_name), end='\n', file=outfile)
        print('-'*width, end='\n', file=outfile)
        print(self.pretty_message(), end='\n', file=outfile)
        print('-'*width, end='\n', file=outfile)
    
    def pretty_message(self):
        """ Return a message to be printed by pretty_print. Returns an empyt string if not overriden. """
        return ''
        


class FailedTestResult(TestResult):
    """
    A basic implementation of TestResult for failed tests.
    """
    def __init__(self, testcase, status=TestResult._fail, reason=None):
        super(FailedTestResult, self).__init__(testcase, status)
        self.reason = reason
    
    def pprint_dict(self):
        """
        Return a dict that is ideal for passing to pprint.
        """
        ppdict = super(FailedTestResult, self).pprint_dict()
        ppdict['reason'] = self.reason
        return ppdict
    
    def pretty_message(self):
        return self.reason


class CompoundTestResult(TestResult):
    """
    A TestResult returned by running a sequencial test case
    """
    def __init__(self, testcase, results):
        super(CompoundTestResult, self).__init__(testcase, None)
        self.sub_results = results
        status_list = [r.status for r in results]
        if TestResult._fail in status_list:
            self.status = TestResult._fail
        elif TestResult._error in status_list:
            self.status = TestResult._error
        elif TestResult._pass in status_list:
            self.status = TestResult._pass
        else:
            self.status = TestResult._skipped
    
    def pprint_dict(self):
        """
        Returns a dict that is ideal for passing to pprint.
        """
        ppdict = super(CompoundTestResult, self).pprint_dict()
        ppdict['sub_results'] = list(s.pprint_dict() for s in self.sub_results)
        return ppdict
    
    def pretty_message(self):
        return '\n'.join(
                "{}: {}{}".format(
                    st.status,
                    st.testcase_name,
                    '\n' + st.pretty_message() if st.status in [TestResult._fail, TestResult._error] else '')
                for st in self.sub_results)


class SubTestResult(TestResult):
    """
    A TestResult for a subtest in a sequencial test case.
    """
    def __init__(self, subtest_name, inner_result):
        """
        @param subtest_name The name of the subtest.
        @param inner_result The result returned from running the subtest.
        """
        super(SubTestResult, self).__init__(inner_result.testcase, inner_result.status)
        self.inner_result = inner_result
    
    def pprint_dict(self):
        """
        Return a dict that is ideal for passing to pprint.
        """
        ppdict = super(CompoundTestResult, self).pprint_dict()
        ppdict['inner_result'] = self.inner_result.pprint_dict()
        return ppdict


class UnhandledExceptionTestResult(TestResult):
    """
    A TestResult returned for exceptions that the test case failed to handle.
    """
    def __init__(self, testcase, status, exc_type, exc_value, exc_traceback):
        super(UnhandledExceptionTestResult, self).__init__(testcase, status)
        self.exception_type = exc_type
        self.exception_value = exc_value
        if not exc_traceback is None:
            sio = util.StringIO()
            traceback.print_exception(self.exception_type, self.exception_value, exc_traceback, file=sio)
            self.exception_message = sio.getvalue()
        else:
            self.exception_message = "{}: {}".format(str(exc_type), str(exc_value))
    
    def pprint_dict(self):
        """
        Return a dict that is ideal for passing to pprint.
        """
        ppdict = super(UnhandledExceptionTestResult, self).pprint_dict()
        ppdict['exception_type'] = self.exception_type
        ppdict['exception_value'] = self.exception_value
        ppdict['exception_message'] = self.exception_message
        return ppdict
    
    def pretty_message(self):
        return self.exception_message


class TestCase(object):
    """
    Base class for all test cases
    """
    def __init__(self, name=None):
        """
        @param name A unique test case name.
        """
        self.name = name    
    
    def setUp(self):
        """
        Called imediately before a testcase is executed.
        """
        pass
    
    def run(self):
        """
        Run the test case, and return its result.
        """
        raise NotImplementedError
    
    def tearDown(self):
        """
        Called imediately after a testcase is executed.
        """
        pass
    
    def catch(self, exc):
        """
        Called when run raises an exception. If the test case
        knows how to handle it, it should return it's own result or None.
        Otherwise, return the original exception.
        """
        return exc
    
    def setresult(self, test_result):
        """
        Called after a test issues a result and before tearDown is called.
        """
        self.test_result = test_result
    
    def make_pass(self, result_type=TestResult, *args, **kwargs):
        """
        Make a passed result for this testcase.
        """
        return TestResult.make_pass(result_type, self, *args, **kwargs)
    
    def make_fail(self, result_type=FailedTestResult, *args, **kwargs):
        """
        Make a failed result for this testcase.
        """
        return TestResult.make_fail(result_type, self, *args, **kwargs)


class SequencialTestCase(TestCase):
    """
    A test case that executes a sequence of subtests until
    one fails.
    """
    def __init__(self, name):
        super(SequencialTestCase, self).__init__(name)
        self.tests = []
    
    def add_subtest(self, subtest_name, subtest_func):
        """
        Add a subtest.
        """
        self.tests.append((subtest_name, subtest_func))
    
    def run(self):
        return CompoundTestResult(self, list(self._runall()))
    
    def _runall(self):
        return _rungen([SubTestCase(name, func) for name, func in self.tests], failfast=True)


class SubTestCase(TestCase):
    """
    A subtest of a sequncial test.
    """
    def __init__(self, name, func):
        super(SubTestCase, self).__init__(name)
        self.run = lambda: func(self)


def run(tclist, failfast=False):
    """
    Run all test cases in tclist and return a list of thier results.
    """
    return list(_rungen(tclist, failfast))

def _rungen(tclist, failfast=False):
    """
    A generator for running tests internally.
    """
    for tc in tclist:
        result = None
        tc.setUp()
        try:
            result = _result(tc.run(), tc)
        except Exception as ex:
            result = _result(tc.catch(ex), tc)
        tc.tearDown()
        yield result
        if failfast and (result.failed() or result.errored()):
            break

def _result(res, tc):
    """
    Convert res to a TestResult object.
    If res is a TestResult object, give it back.
    If res is an Exception, return an UnandledExceptionTestResult.
    If res is something else, discard it and return a passed TestResult.
    """
    if isinstance(res, TestResult):
        return res
    elif isinstance(res, Exception):
        logging.info('uncaught exception: {}'.format(str(res)))
        return TestResult.make_error(UnhandledExceptionTestResult, tc, *(sys.exc_info()))
    else:
        return TestResult.make_pass(TestResult, tc)

def pprint_results(result_iter, outfile=sys.stdout):
    """
    Print pprint version of test results to a file-like object.
    @param result_iter An iterator of results to print.
    @param outfile An opened file-like object to print to (defaults to stdout).
    """
    status_func = lambda r: r.status
    result_iter = sorted(result_iter, key=status_func)
    status_dict = dict(iter((k, list(map(lambda tc: tc.pprint_dict(), g))) for k, g in itertools.groupby(result_iter, status_func)))
    pprint.pprint(status_dict, stream=outfile)

def pretty_print_results(
        result_iter,
        count_by_status=True, exclude_passed=True, exclude_skipped=True, exclude_failed=False,
        exclude_errored=False, sort_by_status=True, width=60, outfile=sys.stdout):
    """
    Print iterator of TestResults in a human readable format to a file-like object.
    @param result_iter An iterator of TestResult objects to print.
    @param count_by_status Print the number of tests for each status (defaults to True).
    @param exclude_passed Exclude passed test results from printing (defaults to True).
    @param exclude_skipped Exclude skipped test results from printing (defaults to True).
    @param exclude_failed Exclude failed test results from printing (defaults to False).
    @param exclude_errored Exclude errored test results from printing (defaults to False).
    @param sort_by_status Print test results in order of status: passed, errored, failed, then skipped (defaults to True).
    @param width Printing width (defaults to 60).
    @param outfile A file-like object to print to (defaults to stdout).
    """
    result_list = list(result_iter)
    status_func = lambda r: r.status
    if sort_by_status:
        #TODO: printing order
        result_iter = sorted(result_iter, key=status_func)
    
    if count_by_status:
        print('Passed: {}'.format(len([tr for tr in result_list if tr.passed()])), file=outfile)
        print('Errors: {}'.format(len([tr for tr in result_list if tr.errored()])), file=outfile)
        print('Failed: {}'.format(len([tr for tr in result_list if tr.failed()])), file=outfile)
        print('Skipped: {}'.format(len([tr for tr in result_list if tr.skipped()])), file=outfile)
    #TODO: something that doesn't expose TestResult._*
    print_status = set(itertools.compress([TestResult._pass, TestResult._error, TestResult._fail, TestResult._skipped],
            map(lambda n: not n, [exclude_passed, exclude_errored, exclude_failed, exclude_skipped])))
    for tr in (r for r in result_list if r.status in print_status):
        tr.pretty_print(width=width, outfile=outfile)