398 lines
14 KiB
Python
398 lines
14 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
|
||
|
|
"""unittest-xml-reporting is a PyUnit-based TestRunner that can export test
|
||
|
|
results to XML files that can be consumed by a wide range of tools, such as
|
||
|
|
build systems, IDEs and Continuous Integration servers.
|
||
|
|
|
||
|
|
This module provides the XMLTestRunner class, which is heavily based on the
|
||
|
|
default TextTestRunner. This makes the XMLTestRunner very simple to use.
|
||
|
|
|
||
|
|
The script below, adapted from the unittest documentation, shows how to use
|
||
|
|
XMLTestRunner in a very simple way. In fact, the only difference between this
|
||
|
|
script and the original one is the last line:
|
||
|
|
|
||
|
|
import random
|
||
|
|
import unittest
|
||
|
|
import xmlrunner
|
||
|
|
|
||
|
|
class TestSequenceFunctions(unittest.TestCase):
|
||
|
|
def setUp(self):
|
||
|
|
self.seq = range(10)
|
||
|
|
|
||
|
|
def test_shuffle(self):
|
||
|
|
# make sure the shuffled sequence does not lose any elements
|
||
|
|
random.shuffle(self.seq)
|
||
|
|
self.seq.sort()
|
||
|
|
self.assertEqual(self.seq, range(10))
|
||
|
|
|
||
|
|
def test_choice(self):
|
||
|
|
element = random.choice(self.seq)
|
||
|
|
self.assertTrue(element in self.seq)
|
||
|
|
|
||
|
|
def test_sample(self):
|
||
|
|
self.assertRaises(ValueError, random.sample, self.seq, 20)
|
||
|
|
for element in random.sample(self.seq, 5):
|
||
|
|
self.assertTrue(element in self.seq)
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports'))
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import absolute_import
|
||
|
|
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from unittest import TestResult, TextTestResult, TextTestRunner
|
||
|
|
import xml.dom.minidom
|
||
|
|
try:
|
||
|
|
from StringIO import StringIO
|
||
|
|
except ImportError:
|
||
|
|
from io import StringIO # doesn't accept 'str' in Py2
|
||
|
|
|
||
|
|
|
||
|
|
class XMLDocument(xml.dom.minidom.Document):
|
||
|
|
def createCDATAOrText(self, data):
|
||
|
|
if ']]>' in data:
|
||
|
|
return self.createTextNode(data)
|
||
|
|
return self.createCDATASection(data)
|
||
|
|
|
||
|
|
|
||
|
|
class _TestInfo(object):
|
||
|
|
"""This class is used to keep useful information about the execution of a
|
||
|
|
test method.
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Possible test outcomes
|
||
|
|
(SUCCESS, FAILURE, ERROR) = range(3)
|
||
|
|
|
||
|
|
def __init__(self, test_result, test_method, outcome=SUCCESS, err=None):
|
||
|
|
"Create a new instance of _TestInfo."
|
||
|
|
self.test_result = test_result
|
||
|
|
self.test_method = test_method
|
||
|
|
self.outcome = outcome
|
||
|
|
self.err = err
|
||
|
|
self.stdout = test_result.stdout and test_result.stdout.getvalue().strip() or ''
|
||
|
|
self.stderr = test_result.stdout and test_result.stderr.getvalue().strip() or ''
|
||
|
|
|
||
|
|
def get_elapsed_time(self):
|
||
|
|
"""Return the time that shows how long the test method took to
|
||
|
|
execute.
|
||
|
|
"""
|
||
|
|
return self.test_result.stop_time - self.test_result.start_time
|
||
|
|
|
||
|
|
def get_description(self):
|
||
|
|
"Return a text representation of the test method."
|
||
|
|
return self.test_result.getDescription(self.test_method)
|
||
|
|
|
||
|
|
def get_error_info(self):
|
||
|
|
"""Return a text representation of an exception thrown by a test
|
||
|
|
method.
|
||
|
|
"""
|
||
|
|
if not self.err:
|
||
|
|
return ''
|
||
|
|
return self.test_result._exc_info_to_string(
|
||
|
|
self.err, self.test_method)
|
||
|
|
|
||
|
|
|
||
|
|
class _XMLTestResult(TextTestResult):
|
||
|
|
"""A test result class that can express test results in a XML report.
|
||
|
|
|
||
|
|
Used by XMLTestRunner.
|
||
|
|
"""
|
||
|
|
def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1,
|
||
|
|
elapsed_times=True):
|
||
|
|
"Create a new instance of _XMLTestResult."
|
||
|
|
TextTestResult.__init__(self, stream, descriptions, verbosity)
|
||
|
|
self.successes = []
|
||
|
|
self.callback = None
|
||
|
|
self.elapsed_times = elapsed_times
|
||
|
|
self.output_patched = False
|
||
|
|
|
||
|
|
def _prepare_callback(self, test_info, target_list, verbose_str,
|
||
|
|
short_str):
|
||
|
|
"""Append a _TestInfo to the given target list and sets a callback
|
||
|
|
method to be called by stopTest method.
|
||
|
|
"""
|
||
|
|
target_list.append(test_info)
|
||
|
|
def callback():
|
||
|
|
"""This callback prints the test method outcome to the stream,
|
||
|
|
as well as the elapsed time.
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Ignore the elapsed times for a more reliable unit testing
|
||
|
|
if not self.elapsed_times:
|
||
|
|
self.start_time = self.stop_time = 0
|
||
|
|
|
||
|
|
if self.showAll:
|
||
|
|
self.stream.writeln('(%.3fs) %s' % \
|
||
|
|
(test_info.get_elapsed_time(), verbose_str))
|
||
|
|
elif self.dots:
|
||
|
|
self.stream.write(short_str)
|
||
|
|
self.callback = callback
|
||
|
|
|
||
|
|
def _patch_standard_output(self):
|
||
|
|
"""Replace the stdout and stderr streams with string-based streams
|
||
|
|
in order to capture the tests' output.
|
||
|
|
"""
|
||
|
|
if not self.output_patched:
|
||
|
|
(self.old_stdout, self.old_stderr) = (sys.stdout, sys.stderr)
|
||
|
|
self.output_patched = True
|
||
|
|
(sys.stdout, sys.stderr) = (self.stdout, self.stderr) = \
|
||
|
|
(StringIO(), StringIO())
|
||
|
|
|
||
|
|
def _restore_standard_output(self):
|
||
|
|
"Restore the stdout and stderr streams."
|
||
|
|
(sys.stdout, sys.stderr) = (self.old_stdout, self.old_stderr)
|
||
|
|
self.output_patched = False
|
||
|
|
|
||
|
|
def startTest(self, test):
|
||
|
|
"Called before execute each test method."
|
||
|
|
self._patch_standard_output()
|
||
|
|
self.start_time = time.time()
|
||
|
|
TestResult.startTest(self, test)
|
||
|
|
|
||
|
|
if self.showAll:
|
||
|
|
self.stream.write(' ' + self.getDescription(test))
|
||
|
|
self.stream.write(" ... ")
|
||
|
|
|
||
|
|
def stopTest(self, test):
|
||
|
|
"Called after execute each test method."
|
||
|
|
self._restore_standard_output()
|
||
|
|
TextTestResult.stopTest(self, test)
|
||
|
|
self.stop_time = time.time()
|
||
|
|
|
||
|
|
if self.callback and callable(self.callback):
|
||
|
|
self.callback()
|
||
|
|
self.callback = None
|
||
|
|
|
||
|
|
def addSuccess(self, test):
|
||
|
|
"Called when a test executes successfully."
|
||
|
|
self._prepare_callback(_TestInfo(self, test),
|
||
|
|
self.successes, 'OK', '.')
|
||
|
|
|
||
|
|
def addFailure(self, test, err):
|
||
|
|
"Called when a test method fails."
|
||
|
|
self._prepare_callback(_TestInfo(self, test, _TestInfo.FAILURE, err),
|
||
|
|
self.failures, 'FAIL', 'F')
|
||
|
|
|
||
|
|
def addError(self, test, err):
|
||
|
|
"Called when a test method raises an error."
|
||
|
|
self._prepare_callback(_TestInfo(self, test, _TestInfo.ERROR, err),
|
||
|
|
self.errors, 'ERROR', 'E')
|
||
|
|
|
||
|
|
def printErrorList(self, flavour, errors):
|
||
|
|
"Write some information about the FAIL or ERROR to the stream."
|
||
|
|
for test_info in errors:
|
||
|
|
if isinstance(test_info, tuple):
|
||
|
|
test_info, exc_info = test_info
|
||
|
|
|
||
|
|
try:
|
||
|
|
t = test_info.get_elapsed_time()
|
||
|
|
except AttributeError:
|
||
|
|
t = 0
|
||
|
|
try:
|
||
|
|
descr = test_info.get_description()
|
||
|
|
except AttributeError:
|
||
|
|
try:
|
||
|
|
descr = test_info.getDescription()
|
||
|
|
except AttributeError:
|
||
|
|
descr = str(test_info)
|
||
|
|
try:
|
||
|
|
err_info = test_info.get_error_info()
|
||
|
|
except AttributeError:
|
||
|
|
err_info = str(test_info)
|
||
|
|
|
||
|
|
self.stream.writeln(self.separator1)
|
||
|
|
self.stream.writeln('%s [%.3fs]: %s' % (flavour, t, descr))
|
||
|
|
self.stream.writeln(self.separator2)
|
||
|
|
self.stream.writeln('%s' % err_info)
|
||
|
|
|
||
|
|
def _get_info_by_testcase(self):
|
||
|
|
"""This method organizes test results by TestCase module. This
|
||
|
|
information is used during the report generation, where a XML report
|
||
|
|
will be generated for each TestCase.
|
||
|
|
"""
|
||
|
|
tests_by_testcase = {}
|
||
|
|
|
||
|
|
for tests in (self.successes, self.failures, self.errors):
|
||
|
|
for test_info in tests:
|
||
|
|
if not isinstance(test_info, _TestInfo):
|
||
|
|
print("Unexpected test result type: %r" % (test_info,))
|
||
|
|
continue
|
||
|
|
testcase = type(test_info.test_method)
|
||
|
|
|
||
|
|
# Ignore module name if it is '__main__'
|
||
|
|
module = testcase.__module__ + '.'
|
||
|
|
if module == '__main__.':
|
||
|
|
module = ''
|
||
|
|
testcase_name = module + testcase.__name__
|
||
|
|
|
||
|
|
if testcase_name not in tests_by_testcase:
|
||
|
|
tests_by_testcase[testcase_name] = []
|
||
|
|
tests_by_testcase[testcase_name].append(test_info)
|
||
|
|
|
||
|
|
return tests_by_testcase
|
||
|
|
|
||
|
|
def _report_testsuite(suite_name, tests, xml_document):
|
||
|
|
"Appends the testsuite section to the XML document."
|
||
|
|
testsuite = xml_document.createElement('testsuite')
|
||
|
|
xml_document.appendChild(testsuite)
|
||
|
|
|
||
|
|
testsuite.setAttribute('name', str(suite_name))
|
||
|
|
testsuite.setAttribute('tests', str(len(tests)))
|
||
|
|
|
||
|
|
testsuite.setAttribute('time', '%.3f' %
|
||
|
|
sum([e.get_elapsed_time() for e in tests]))
|
||
|
|
|
||
|
|
failures = len([1 for e in tests if e.outcome == _TestInfo.FAILURE])
|
||
|
|
testsuite.setAttribute('failures', str(failures))
|
||
|
|
|
||
|
|
errors = len([1 for e in tests if e.outcome == _TestInfo.ERROR])
|
||
|
|
testsuite.setAttribute('errors', str(errors))
|
||
|
|
|
||
|
|
return testsuite
|
||
|
|
|
||
|
|
_report_testsuite = staticmethod(_report_testsuite)
|
||
|
|
|
||
|
|
def _report_testcase(suite_name, test_result, xml_testsuite, xml_document):
|
||
|
|
"Appends a testcase section to the XML document."
|
||
|
|
testcase = xml_document.createElement('testcase')
|
||
|
|
xml_testsuite.appendChild(testcase)
|
||
|
|
|
||
|
|
testcase.setAttribute('classname', str(suite_name))
|
||
|
|
testcase.setAttribute('name', test_result.test_method.shortDescription()
|
||
|
|
or getattr(test_result.test_method, '_testMethodName',
|
||
|
|
str(test_result.test_method)))
|
||
|
|
testcase.setAttribute('time', '%.3f' % test_result.get_elapsed_time())
|
||
|
|
|
||
|
|
if (test_result.outcome != _TestInfo.SUCCESS):
|
||
|
|
elem_name = ('failure', 'error')[test_result.outcome-1]
|
||
|
|
failure = xml_document.createElement(elem_name)
|
||
|
|
testcase.appendChild(failure)
|
||
|
|
|
||
|
|
failure.setAttribute('type', str(test_result.err[0].__name__))
|
||
|
|
failure.setAttribute('message', str(test_result.err[1]))
|
||
|
|
|
||
|
|
error_info = test_result.get_error_info()
|
||
|
|
failureText = xml_document.createCDATAOrText(error_info)
|
||
|
|
failure.appendChild(failureText)
|
||
|
|
|
||
|
|
_report_testcase = staticmethod(_report_testcase)
|
||
|
|
|
||
|
|
def _report_output(test_runner, xml_testsuite, xml_document, stdout, stderr):
|
||
|
|
"Appends the system-out and system-err sections to the XML document."
|
||
|
|
systemout = xml_document.createElement('system-out')
|
||
|
|
xml_testsuite.appendChild(systemout)
|
||
|
|
|
||
|
|
systemout_text = xml_document.createCDATAOrText(stdout)
|
||
|
|
systemout.appendChild(systemout_text)
|
||
|
|
|
||
|
|
systemerr = xml_document.createElement('system-err')
|
||
|
|
xml_testsuite.appendChild(systemerr)
|
||
|
|
|
||
|
|
systemerr_text = xml_document.createCDATAOrText(stderr)
|
||
|
|
systemerr.appendChild(systemerr_text)
|
||
|
|
|
||
|
|
_report_output = staticmethod(_report_output)
|
||
|
|
|
||
|
|
def generate_reports(self, test_runner):
|
||
|
|
"Generates the XML reports to a given XMLTestRunner object."
|
||
|
|
all_results = self._get_info_by_testcase()
|
||
|
|
|
||
|
|
if type(test_runner.output) == str and not \
|
||
|
|
os.path.exists(test_runner.output):
|
||
|
|
os.makedirs(test_runner.output)
|
||
|
|
|
||
|
|
for suite, tests in all_results.items():
|
||
|
|
doc = XMLDocument()
|
||
|
|
|
||
|
|
# Build the XML file
|
||
|
|
testsuite = _XMLTestResult._report_testsuite(suite, tests, doc)
|
||
|
|
stdout, stderr = [], []
|
||
|
|
for test in tests:
|
||
|
|
_XMLTestResult._report_testcase(suite, test, testsuite, doc)
|
||
|
|
if test.stdout:
|
||
|
|
stdout.extend(['*****************', test.get_description(), test.stdout])
|
||
|
|
if test.stderr:
|
||
|
|
stderr.extend(['*****************', test.get_description(), test.stderr])
|
||
|
|
_XMLTestResult._report_output(test_runner, testsuite, doc,
|
||
|
|
'\n'.join(stdout), '\n'.join(stderr))
|
||
|
|
xml_content = doc.toprettyxml(indent='\t')
|
||
|
|
|
||
|
|
if type(test_runner.output) is str:
|
||
|
|
report_file = open('%s%sTEST-%s.xml' % \
|
||
|
|
(test_runner.output, os.sep, suite), 'w')
|
||
|
|
try:
|
||
|
|
report_file.write(xml_content)
|
||
|
|
finally:
|
||
|
|
report_file.close()
|
||
|
|
else:
|
||
|
|
# Assume that test_runner.output is a stream
|
||
|
|
test_runner.output.write(xml_content)
|
||
|
|
|
||
|
|
|
||
|
|
class XMLTestRunner(TextTestRunner):
|
||
|
|
"""A test runner class that outputs the results in JUnit like XML files.
|
||
|
|
"""
|
||
|
|
def __init__(self, output='.', stream=None, descriptions=True, verbose=False, elapsed_times=True):
|
||
|
|
"Create a new instance of XMLTestRunner."
|
||
|
|
if stream is None:
|
||
|
|
stream = sys.stderr
|
||
|
|
verbosity = (1, 2)[verbose]
|
||
|
|
TextTestRunner.__init__(self, stream, descriptions, verbosity)
|
||
|
|
self.output = output
|
||
|
|
self.elapsed_times = elapsed_times
|
||
|
|
|
||
|
|
def _make_result(self):
|
||
|
|
"""Create the TestResult object which will be used to store
|
||
|
|
information about the executed tests.
|
||
|
|
"""
|
||
|
|
return _XMLTestResult(self.stream, self.descriptions, \
|
||
|
|
self.verbosity, self.elapsed_times)
|
||
|
|
|
||
|
|
def run(self, test):
|
||
|
|
"Run the given test case or test suite."
|
||
|
|
# Prepare the test execution
|
||
|
|
result = self._make_result()
|
||
|
|
|
||
|
|
# Print a nice header
|
||
|
|
self.stream.writeln()
|
||
|
|
self.stream.writeln('Running tests...')
|
||
|
|
self.stream.writeln(result.separator2)
|
||
|
|
|
||
|
|
# Execute tests
|
||
|
|
start_time = time.time()
|
||
|
|
test(result)
|
||
|
|
stop_time = time.time()
|
||
|
|
time_taken = stop_time - start_time
|
||
|
|
|
||
|
|
# Generate reports
|
||
|
|
self.stream.writeln()
|
||
|
|
self.stream.writeln('Generating XML reports...')
|
||
|
|
result.generate_reports(self)
|
||
|
|
|
||
|
|
# Print results
|
||
|
|
result.printErrors()
|
||
|
|
self.stream.writeln(result.separator2)
|
||
|
|
run = result.testsRun
|
||
|
|
self.stream.writeln("Ran %d test%s in %.3fs" %
|
||
|
|
(run, run != 1 and "s" or "", time_taken))
|
||
|
|
self.stream.writeln()
|
||
|
|
|
||
|
|
# Error traces
|
||
|
|
if not result.wasSuccessful():
|
||
|
|
self.stream.write("FAILED (")
|
||
|
|
failed, errored = (len(result.failures), len(result.errors))
|
||
|
|
if failed:
|
||
|
|
self.stream.write("failures=%d" % failed)
|
||
|
|
if errored:
|
||
|
|
if failed:
|
||
|
|
self.stream.write(", ")
|
||
|
|
self.stream.write("errors=%d" % errored)
|
||
|
|
self.stream.writeln(")")
|
||
|
|
else:
|
||
|
|
self.stream.writeln("OK")
|
||
|
|
|
||
|
|
return result
|