diff --git a/README.rst b/README.rst index 443203b..9420e48 100644 --- a/README.rst +++ b/README.rst @@ -6,8 +6,8 @@ Use `pytest `_ runner to discover and execu |python| |version| |downloads| |ci| |coverage| -Supports both `Google Test `_ and -`Boost::Test `_: +Supports `Google Test `_ , +`Boost::Test `_ and `QTest `_: .. image:: https://raw.githubusercontent.com/pytest-dev/pytest-cpp/master/images/screenshot.png @@ -43,7 +43,7 @@ Usage Once installed, when py.test runs it will search and run tests founds in executable files, detecting if the suites are -Google or Boost tests automatically. +Google, Boost or Qt tests automatically. You can configure which files are tested for suites by using the ``cpp_files`` ini configuration:: diff --git a/pytest_cpp/plugin.py b/pytest_cpp/plugin.py index d544ff4..0c60b95 100644 --- a/pytest_cpp/plugin.py +++ b/pytest_cpp/plugin.py @@ -8,7 +8,9 @@ from pytest_cpp.google import GoogleTestFacade -FACADES = [GoogleTestFacade, BoostTestFacade] +from pytest_cpp.qt import QTestLibFacade + +FACADES = [GoogleTestFacade, BoostTestFacade, QTestLibFacade] DEFAULT_MASKS = ('test_*', '*_test') diff --git a/pytest_cpp/qt.py b/pytest_cpp/qt.py new file mode 100644 index 0000000..dd926e5 --- /dev/null +++ b/pytest_cpp/qt.py @@ -0,0 +1,133 @@ +import os +import subprocess +import tempfile +import io +import shutil +from pytest_cpp.error import CppTestFailure +import xml.etree.ElementTree as ET + + +class QTestLibFacade(object): + """ + Facade for QTestLib. + """ + + @classmethod + def is_test_suite(cls, executable): + try: + output = subprocess.check_output([executable, '-help'], + stderr=subprocess.STDOUT, + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + return False + else: + return '-datatags' in output + + def list_tests(self, executable): + # unfortunately boost doesn't provide us with a way to list the tests + # inside the executable, so the test_id is a dummy placeholder :( + return [os.path.basename(os.path.splitext(executable)[0])] + + def run_test(self, executable, test_id): + def read_file(name): + try: + with io.open(name) as f: + return f.read() + except IOError: + return None + + temp_dir = tempfile.mkdtemp() + log_xml = os.path.join(temp_dir, 'log.xml') + args = [ + executable, + "-o", + "{xml_file},xml".format(xml_file=log_xml), + ] + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout, _ = p.communicate() + + num_reports = len(os.listdir(temp_dir)) + if num_reports > 1: + self.merge_xml_report(temp_dir) + log_xml = os.path.join(temp_dir, "result-merged.xml") + log = read_file(log_xml) + elif num_reports == 1: + log_xml = os.path.join(temp_dir, os.listdir(temp_dir)[0]) + log = read_file(log_xml) + else: + log_xml = log = None + + if p.returncode < 0 and p.returncode not in(-6, ): + msg = ('Internal Error: calling {executable} ' + 'for test {test_id} failed (returncode={returncode}):\n' + 'output:{stdout}\n' + 'log:{log}\n') + failure = QTestFailure( + '', + linenum=0, + contents=msg.format(executable=executable, + test_id=test_id, + stdout=stdout, + log=log, + returncode=p.returncode)) + return [failure] + + results = self._parse_log(log=log_xml) + shutil.rmtree(temp_dir) + + if results: + return results + + def merge_xml_report(self, temp_dir): + matches = [] + for root, dirnames, filenames in os.walk(temp_dir): + for filename in filenames: + if filename.endswith('.xml'): + matches.append(os.path.join(root, filename)) + + cases = [] + suites = [] + + for file_name in matches: + tree = ET.parse(file_name) + test_suite = tree.getroot() + cases.append(test_suite.getchildren()) + suites.append(test_suite) + + new_root = ET.Element('testsuites') + + for suite in suites: + new_root.append(suite) + + new_tree = ET.ElementTree(new_root) + new_tree.write(os.path.join(temp_dir, "result-merged.xml"), + encoding="UTF-8", + xml_declaration=True) + + def _parse_log(self, log): + failed_suites = [] + tree = ET.parse(log) + root = tree.getroot() + if log: + for suite in root: + failed_cases = [case for case in root.iter('Incident') if case.get('type') != "pass"] + if failed_cases: + failed_suites = [] + for case in failed_cases: + failed_suites.append(QTestFailure(case.attrib['file'], int(case.attrib['line']), case.find('Description').text)) + return failed_suites + + +class QTestFailure(CppTestFailure): + + def __init__(self, filename, linenum, contents): + self.filename = filename + self.linenum = linenum + self.lines = contents.splitlines() + + def get_lines(self): + m = ('red', 'bold') + return [(x, m) for x in self.lines] + + def get_file_reference(self): + return self.filename, self.linenum diff --git a/tests/SConstruct b/tests/SConstruct index 40ba0b3..9944beb 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -1,6 +1,13 @@ import os import sys +# In order to build the tests for QT, you need to: +# - Download and install qt from https://www.qt.io/download/ +# - Set the QT5DIR variable below +# - Install the Scons Qt5 tools from https://bitbucket.org/dirkbaechle/scons_qt5 +# First install option worked for me (installing it directly in the pytest-cpp/tests directory) +ENABLE_QT_TEST = False + if sys.platform.startswith('win'): CCFLAGS = ['/EHsc', '/nologo'] LIBS = [] @@ -9,19 +16,40 @@ else: LIBS = ['pthread'] env = Environment( - CPPPATH=os.environ.get('INCLUDE'), + CPPPATH=os.environ.get('INCLUDE', []), CCFLAGS=CCFLAGS, - LIBPATH=os.environ.get('LIBPATH'), + LIBPATH=os.environ.get('LIBPATH', []), LIBS=LIBS, ) + +# google test env genv = env.Clone(LIBS=['gtest'] + LIBS) +if ENABLE_QT_TEST: + # qt5 env + QT5DIR = "/opt/Qt5.7.0/5.7/gcc_64" + qtenv = env.Clone(QT5DIR=QT5DIR, + CCFLAGS="-std=c++11 -fPIC", + tools=['default','qt5']) + qtenv['ENV']['PKG_CONFIG_PATH'] = os.path.join(QT5DIR, 'lib/pkgconfig') + qtenv.EnableQt5Modules([ + 'QtTest' + ]) + Export('qtenv') + Export('env genv') +# build google test target genv.Program('gtest.cpp') +# build boost target for filename in ('boost_success.cpp', 'boost_failure.cpp', 'boost_error.cpp'): env.Program(filename) +# build qt5 target +if ENABLE_QT_TEST: + for filename in ('qt_success.cpp', 'qt_failure.cpp', 'qt_error.cpp'): + qtenv.Program(filename) + SConscript('acceptance/googletest-samples/SConscript') -SConscript('acceptance/boosttest-samples/SConscript') \ No newline at end of file +SConscript('acceptance/boosttest-samples/SConscript') diff --git a/tests/qt_error.cpp b/tests/qt_error.cpp new file mode 100644 index 0000000..222f950 --- /dev/null +++ b/tests/qt_error.cpp @@ -0,0 +1,22 @@ +#include + +class TestError: public QObject +{ + Q_OBJECT +private slots: + void testErrorOne(); + void testErrorTwo(); +}; + +void TestError::testErrorOne() +{ + throw std::runtime_error("unexpected exception"); +} + +void TestError::testErrorTwo() +{ + throw std::runtime_error("another unexpected exception"); +} + +QTEST_MAIN(TestError) +#include "qt_error.moc" diff --git a/tests/qt_failure.cpp b/tests/qt_failure.cpp new file mode 100644 index 0000000..6e8bfc4 --- /dev/null +++ b/tests/qt_failure.cpp @@ -0,0 +1,22 @@ +#include + +class TestFailure: public QObject +{ + Q_OBJECT +private slots: + void TestFailureOne(); + void TestFailureTwo(); +}; + +void TestFailure::TestFailureOne() +{ + QCOMPARE(2 * 3, 5); +} + +void TestFailure::TestFailureTwo() +{ + QCOMPARE(2 - 1, 0); +} + +QTEST_MAIN(TestFailure) +#include "qt_failure.moc" diff --git a/tests/qt_success.cpp b/tests/qt_success.cpp new file mode 100644 index 0000000..7850d7c --- /dev/null +++ b/tests/qt_success.cpp @@ -0,0 +1,22 @@ +#include + +class TestSuccess: public QObject +{ + Q_OBJECT +private slots: + void testSuccessOne(); + void testSuccessTwo(); +}; + +void TestSuccess::testSuccessOne() +{ + QCOMPARE(2 * 3, 6); +} + +void TestSuccess::testSuccessTwo() +{ + QCOMPARE(3 * 4, 12); +} + +QTEST_MAIN(TestSuccess) +#include "qt_success.moc" diff --git a/tests/test_pytest_cpp.py b/tests/test_pytest_cpp.py index 4648ebe..0e10a20 100644 --- a/tests/test_pytest_cpp.py +++ b/tests/test_pytest_cpp.py @@ -4,6 +4,12 @@ from pytest_cpp.boost import BoostTestFacade from pytest_cpp.error import CppTestFailure, CppFailureRepr from pytest_cpp.google import GoogleTestFacade +from pytest_cpp.qt import QTestLibFacade + + +# To enable QT support plugin tests, set ENABLE_QT_TEST to True in this file +# and in the tests/SConstruct file. +ENABLE_QT_TEST = False def assert_outcomes(result, expected_outcomes): @@ -41,6 +47,7 @@ def get_file_reference(self): 'FooTest.DISABLED_test_disabled' ]), (BoostTestFacade(), 'boost_success', ['boost_success']), + (BoostTestFacade(), 'boost_failure', ['boost_failure']), (BoostTestFacade(), 'boost_error', ['boost_error']), ]) def test_list_tests(facade, name, expected, exes): @@ -143,9 +150,10 @@ def test_google_run(testdir, exes): ('FooTest.DISABLED_test_disabled', 'skipped'), ]) + def test_unknown_error(testdir, exes, mocker): mocker.patch.object(GoogleTestFacade, 'run_test', - side_effect=RuntimeError('unknown error')) + side_effect=RuntimeError('unknown error')) result = testdir.inline_run('-v', exes.get('gtest', 'test_gtest')) rep = result.matchreport('FooTest.test_success', 'pytest_runtest_logreport') assert 'unknown error' in str(rep.longrepr) @@ -154,9 +162,9 @@ def test_unknown_error(testdir, exes, mocker): def test_google_internal_errors(mocker, testdir, exes, tmpdir): mocker.patch.object(GoogleTestFacade, 'is_test_suite', return_value=True) mocker.patch.object(GoogleTestFacade, 'list_tests', - return_value=['FooTest.test_success']) + return_value=['FooTest.test_success']) mocked = mocker.patch.object(subprocess, 'check_output', autospec=True, - return_value='') + return_value='') def raise_error(*args, **kwargs): raise subprocess.CalledProcessError(returncode=100, cmd='') @@ -170,7 +178,7 @@ def raise_error(*args, **kwargs): xml_file = tmpdir.join('results.xml') xml_file.write('') mocker.patch.object(GoogleTestFacade, '_get_temp_xml_filename', - return_value=str(xml_file)) + return_value=str(xml_file)) result = testdir.inline_run('-v', exes.get('gtest', 'test_gtest')) rep = result.matchreport(exes.exe_name('test_gtest'), 'pytest_runtest_logreport') @@ -193,7 +201,7 @@ def mock_popen(mocker, return_code, stdout, stderr): mocked_popen = mocker.MagicMock() mocked_popen.__enter__ = mocked_popen mocked_popen.communicate.return_value = stdout, stderr - mocked_popen.return_code = return_code + mocked_popen.returncode = return_code mocked_popen.poll.return_value = return_code mocker.patch.object(subprocess, 'Popen', return_value=mocked_popen) return mocked_popen @@ -220,7 +228,7 @@ def test_cpp_failure_repr(dummy_failure): def test_cpp_files_option(testdir, exes): exes.get('boost_success') exes.get('gtest') - + result = testdir.inline_run('--collect-only') reps = result.getreports() assert len(reps) == 1 @@ -236,9 +244,9 @@ def test_cpp_files_option(testdir, exes): def test_passing_files_directly_in_command_line(testdir, exes): - f = exes.get('boost_success') - result = testdir.runpytest(f) - result.stdout.fnmatch_lines(['*1 passed*']) + boost_exe = exes.get('boost_success') + result_boost = testdir.runpytest(boost_exe) + result_boost.stdout.fnmatch_lines(['*1 passed*']) class TestError: @@ -264,3 +272,91 @@ def test_get_code_context_around_line(self, tmpdir): invalid = str(tmpdir.join('invalid')) assert error.get_code_context_around_line(invalid, 10) == [] + + +if ENABLE_QT_TEST: + @pytest.mark.parametrize('facade, name, expected', [ + (QTestLibFacade(), 'qt_success', ['qt_success']), + (QTestLibFacade(), 'qt_failure', ['qt_failure']), + (QTestLibFacade(), 'qt_error', ['qt_error']) + ]) + def test_qt_list_tests(facade, name, expected, exes): + obtained = facade.list_tests(exes.get(name)) + assert obtained == expected + + @pytest.mark.parametrize('facade, name, other_name', [ + (QTestLibFacade(), 'qt_success', 'gtest'), + ]) + def test_qt_is_test_suite(facade, name, other_name, exes, tmpdir): + assert facade.is_test_suite(exes.get(name)) + assert not facade.is_test_suite(exes.get(other_name)) + tmpdir.ensure('foo.txt') + assert not facade.is_test_suite(str(tmpdir.join('foo.txt'))) + + @pytest.mark.parametrize('facade, name, test_id', [ + (QTestLibFacade(), 'qt_success', ''), + ]) + def test_qt_success(facade, name, test_id, exes): + assert facade.run_test(exes.get(name), test_id) is None + + def test_qt_failure(exes): + facade = QTestLibFacade() + failures = facade.run_test(exes.get('qt_failure'), '') + assert len(failures) == 2 + + fail1 = failures[0] + colors = ('red', 'bold') + assert fail1.get_lines() == [('Compared values are not the same', colors), (' Actual (2 * 3): 6', colors), (' Expected (5) : 5', colors)] + assert fail1.get_file_reference() == ('qt_failure.cpp', 13) + + def test_qt_error(exes): + facade = QTestLibFacade() + failures = facade.run_test(exes.get('qt_error'), '') + assert len(failures) == 1 # qt abort at first unhandled exception + + fail1 = failures[0] + colors = ('red', 'bold') + + assert fail1.get_lines() == [ + ('Caught unhandled exception', colors)] + + def test_qt_run(testdir, exes): + all_names = ['qt_success', 'qt_error', 'qt_failure'] + all_files = [exes.get(n, 'test_' + n) for n in all_names] + result = testdir.inline_run('-v', *all_files) + assert_outcomes(result, [ + ('test_qt_success', 'passed'), + ('test_qt_error', 'failed'), + ('test_qt_failure', 'failed'), + ]) + + def test_qt_internal_error(testdir, exes, mocker): + exe = exes.get('qt_success', 'test_qt_success') + mock_popen(mocker, return_code=-10, stderr=None, stdout=None) + mocker.patch.object(QTestLibFacade, 'is_test_suite', return_value=True) + mocker.patch.object(GoogleTestFacade, 'is_test_suite', return_value=False) + mocker.patch.object(BoostTestFacade, 'is_test_suite', return_value=False) + result = testdir.inline_run(exe) + rep = result.matchreport(exes.exe_name('test_qt_success'), + 'pytest_runtest_logreport') + assert 'Internal Error:' in str(rep.longrepr) + + def test_qt_cpp_files_option(testdir, exes): + exes.get('qt_success') + + result = testdir.inline_run('--collect-only') + reps = result.getreports() + assert len(reps) == 1 + assert reps[0].result == [] + + testdir.makeini(''' + [pytest] + cpp_files = qt* + ''') + result = testdir.inline_run('--collect-only') + assert len(result.matchreport(exes.exe_name('qt_success')).result) == 1 + + def test_qt_passing_files_directly_in_command_line(testdir, exes): + qt_exe = exes.get('qt_success') + result_qt = testdir.runpytest(qt_exe) + result_qt.stdout.fnmatch_lines(['*1 passed*'])