From ad78e6d4162883699f3a5ea9b68ea6c73173d274 Mon Sep 17 00:00:00 2001 From: KsanaKozlova Date: Wed, 18 Nov 2020 16:35:18 +0300 Subject: [PATCH 1/2] add parrallel runner for tests --- runtests.py | 10 ++++ sdc/runtests.py | 72 +++---------------------- sdc/testing/__init__.py | 63 ++++++++++++++++++++++ sdc/testing/_runtests.py | 114 +++++++++++++++++++++++++++++++++++++++ sdc/tests/__init__.py | 61 +++------------------ 5 files changed, 202 insertions(+), 118 deletions(-) create mode 100644 runtests.py create mode 100644 sdc/testing/__init__.py create mode 100644 sdc/testing/_runtests.py diff --git a/runtests.py b/runtests.py new file mode 100644 index 000000000..3161963e0 --- /dev/null +++ b/runtests.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +import runpy +import os +import sdc +import numba + + +if __name__ == "__main__": + runpy.run_module('sdc.runtests', run_name='__main__') diff --git a/sdc/runtests.py b/sdc/runtests.py index aa0fc14fc..6e1bb501c 100644 --- a/sdc/runtests.py +++ b/sdc/runtests.py @@ -1,68 +1,10 @@ -# ***************************************************************************** -# Copyright (c) 2019-2020, Intel Corporation All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ***************************************************************************** - - -import os -import unittest -import sdc.tests -from sdc.tests.test_basic import get_rank - -""" - Every test in suite can be executed specified times using - desired value for SDC_REPEAT_TEST_NUMBER environment variable. - This can be used to locate scpecific failures occured - on next execution of affected test. - - loadTestsFromModule returns TestSuite obj with _tests member - which contains further TestSuite instanses for each found testCase: - hpat_tests = TestSuite(sdc.tests) - TestSuite(sdc.tests)._tests = [TestSuite(sdc.tests.TestBasic), TestSuite(sdc.tests.TestDataFrame), ...] - TestSuite(sdc.tests.TestBasic)._tests = [TestBasic testMethod=test_array_reduce, ...] -""" - - -def load_tests(loader, tests, pattern): - suite = unittest.TestSuite() - hpat_tests = loader.loadTestsFromModule(sdc.tests) - repeat_test_number = int(os.getenv('SDC_REPEAT_TEST_NUMBER', '1')) - - if repeat_test_number > 1: - for i, test_case in enumerate(hpat_tests): - extended_tests = [] - for test in test_case: - for _ in range(repeat_test_number): - extended_tests.append(test) - hpat_tests._tests[i]._tests = extended_tests - - suite.addTests(hpat_tests) - return suite +from sdc.testing._runtests import _main if __name__ == '__main__': - # initialize MPI to avoid "Attempting to use an MPI routine before initializing MPICH" in any pipeline - get_rank() - - unittest.main() + import sys + # For parallel testing under Windows + from multiprocessing import freeze_support + freeze_support() + + sys.exit(0 if _main(sys.argv) else 1) diff --git a/sdc/testing/__init__.py b/sdc/testing/__init__.py new file mode 100644 index 000000000..0ee5eb0bf --- /dev/null +++ b/sdc/testing/__init__.py @@ -0,0 +1,63 @@ +import os +import sys +import functools +import unittest +import traceback +from fnmatch import fnmatch +from os.path import join, isfile, relpath, normpath, splitext, abspath +import numba + +from numba.testing.main import NumbaTestProgram, SerialSuite, make_tag_decorator + + +def load_testsuite(loader, dir): + """Find tests in 'dir'.""" + try: + suite = unittest.TestSuite() + files = [] + for f in os.listdir(dir): + path = join(dir, f) + if isfile(path) and fnmatch(f, 'test_*.py'): + files.append(f) + elif isfile(join(path, '__init__.py')): + suite.addTests(loader.discover(path, top_level_dir=str(abspath(join(path,"../../.."))))) + for f in files: + # turn 'f' into a filename relative to the toplevel dir... + f = relpath(join(dir, f), loader._top_level_dir) + # ...and translate it to a module name. + f = splitext(normpath(f.replace(os.path.sep, '.')))[0] + suite.addTests(loader.loadTestsFromName(f)) + return suite + except Exception: + traceback.print_exc(file=sys.stderr) + sys.exit(-1) + + + + +def run_tests(argv=None, defaultTest=None, topleveldir=None, + xmloutput=None, verbosity=1, nomultiproc=False): + """ + args + ---- + - xmloutput [str or None] + Path of XML output directory (optional) + - verbosity [int] + Verbosity level of tests output + + Returns the TestResult object after running the test *suite*. + """ + + if xmloutput is not None: + import xmlrunner + runner = xmlrunner.XMLTestRunner(output=xmloutput) + else: + runner = None + prog = NumbaTestProgram(argv=argv, + module=None, + defaultTest=defaultTest, + topleveldir=topleveldir, + testRunner=runner, exit=False, + verbosity=verbosity, + nomultiproc=nomultiproc) + return prog.result diff --git a/sdc/testing/_runtests.py b/sdc/testing/_runtests.py new file mode 100644 index 000000000..a0634c899 --- /dev/null +++ b/sdc/testing/_runtests.py @@ -0,0 +1,114 @@ +import json +import re +import logging + + +def _main(argv, **kwds): + from sdc.testing import run_tests + # This helper function assumes the first element of argv + # is the name of the calling program. + # The 'main' API function is invoked in-process, and thus + # will synthesize that name. + + if '--log' in argv: + logging.basicConfig(level=logging.DEBUG) + argv.remove('--log') + + if '--failed-first' in argv: + # Failed first + argv.remove('--failed-first') + return _FailedFirstRunner().main(argv, kwds) + elif '--last-failed' in argv: + argv.remove('--last-failed') + return _FailedFirstRunner(last_failed=True).main(argv, kwds) + else: + return run_tests(argv, defaultTest='sdc.tests', + **kwds).wasSuccessful() + + +def main(*argv, **kwds): + """keyword arguments are accepted for backward compatibility only. + See `numba.testing.run_tests()` documentation for details.""" + return _main(['
'] + list(argv), **kwds) + + +class _FailedFirstRunner(object): + """ + Test Runner to handle the failed-first (--failed-first) option. + """ + cache_filename = '.runtests_lastfailed' + + def __init__(self, last_failed=False): + self.last_failed = last_failed + + def main(self, argv, kwds): + from sdc.testing import run_tests + prog = argv[0] + argv = argv[1:] + flags = [a for a in argv if a.startswith('-')] + + all_tests, failed_tests = self.find_last_failed(argv) + # Prepare tests to run + if failed_tests: + ft = "There were {} previously failed tests" + print(ft.format(len(failed_tests))) + remaing_tests = [t for t in all_tests + if t not in failed_tests] + if self.last_failed: + tests = list(failed_tests) + else: + tests = failed_tests + remaing_tests + else: + if self.last_failed: + tests = [] + else: + tests = list(all_tests) + + if not tests: + print("No tests to run") + return True + # Run the testsuite + print("Running {} tests".format(len(tests))) + print('Flags', flags) + result = run_tests([prog] + flags + tests, **kwds) + # Update failed tests records only if we have run the all the tests + # last failed. + if len(tests) == result.testsRun: + self.save_failed_tests(result, all_tests) + return result.wasSuccessful() + + def save_failed_tests(self, result, all_tests): + print("Saving failed tests to {}".format(self.cache_filename)) + cache = [] + # Find failed tests + failed = set() + for case in result.errors + result.failures: + failed.add(case[0].id()) + # Build cache + for t in all_tests: + if t in failed: + cache.append(t) + # Write cache + with open(self.cache_filename, 'w') as fout: + json.dump(cache, fout) + + def find_last_failed(self, argv): + from numba.tests.support import captured_output + + # Find all tests + listargv = ['-l'] + [a for a in argv if not a.startswith('-')] + with captured_output("stdout") as stream: + main(*listargv) + + pat = re.compile(r"^(\w+\.)+\w+$") + lines = stream.getvalue().splitlines() + all_tests = [x for x in lines if pat.match(x) is not None] + + try: + fobj = open(self.cache_filename) + except IOError: + failed_tests = [] + else: + with fobj as fin: + failed_tests = json.load(fin) + return all_tests, failed_tests diff --git a/sdc/tests/__init__.py b/sdc/tests/__init__.py index c36067f0f..47389f645 100644 --- a/sdc/tests/__init__.py +++ b/sdc/tests/__init__.py @@ -1,53 +1,8 @@ -# ***************************************************************************** -# Copyright (c) 2019-2020, Intel Corporation All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ***************************************************************************** - - -from sdc.tests.test_basic import * -from sdc.tests.test_series import * -from sdc.tests.test_dataframe import * -from sdc.tests.test_hiframes import * -from .categorical import * - -# from sdc.tests.test_d4p import * -from sdc.tests.test_date import * -from sdc.tests.test_strings import * - -from sdc.tests.test_groupby import * -from sdc.tests.test_join import * -from sdc.tests.test_rolling import * - -from sdc.tests.test_ml import * - -from sdc.tests.test_io import * - -from sdc.tests.test_hpat_jit import * -from sdc.tests.test_indexes import * - -from sdc.tests.test_sdc_numpy import * -from sdc.tests.test_prange_utils import * - -# performance tests -import sdc.tests.tests_perf +from os.path import dirname +from unittest.suite import TestSuite +from sdc.testing import load_testsuite + +def load_tests(loader, tests, pattern): + suite = TestSuite() + suite.addTests(load_testsuite(loader, dirname(__file__))) + return suite From 95679adf4a8f271a09abddab5a0db3215982e7ff Mon Sep 17 00:00:00 2001 From: KsanaKozlova Date: Wed, 18 Nov 2020 16:49:35 +0300 Subject: [PATCH 2/2] fix pep issues --- sdc/runtests.py | 2 -- sdc/testing/__init__.py | 4 +--- sdc/tests/__init__.py | 1 + 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sdc/runtests.py b/sdc/runtests.py index 6e1bb501c..fa39bfb8f 100644 --- a/sdc/runtests.py +++ b/sdc/runtests.py @@ -1,4 +1,3 @@ - from sdc.testing._runtests import _main if __name__ == '__main__': @@ -6,5 +5,4 @@ # For parallel testing under Windows from multiprocessing import freeze_support freeze_support() - sys.exit(0 if _main(sys.argv) else 1) diff --git a/sdc/testing/__init__.py b/sdc/testing/__init__.py index 0ee5eb0bf..d1bb2cd4d 100644 --- a/sdc/testing/__init__.py +++ b/sdc/testing/__init__.py @@ -20,7 +20,7 @@ def load_testsuite(loader, dir): if isfile(path) and fnmatch(f, 'test_*.py'): files.append(f) elif isfile(join(path, '__init__.py')): - suite.addTests(loader.discover(path, top_level_dir=str(abspath(join(path,"../../.."))))) + suite.addTests(loader.discover(path, top_level_dir=str(abspath(join(path, "../../.."))))) for f in files: # turn 'f' into a filename relative to the toplevel dir... f = relpath(join(dir, f), loader._top_level_dir) @@ -33,8 +33,6 @@ def load_testsuite(loader, dir): sys.exit(-1) - - def run_tests(argv=None, defaultTest=None, topleveldir=None, xmloutput=None, verbosity=1, nomultiproc=False): """ diff --git a/sdc/tests/__init__.py b/sdc/tests/__init__.py index 47389f645..46b2b272f 100644 --- a/sdc/tests/__init__.py +++ b/sdc/tests/__init__.py @@ -2,6 +2,7 @@ from unittest.suite import TestSuite from sdc.testing import load_testsuite + def load_tests(loader, tests, pattern): suite = TestSuite() suite.addTests(load_testsuite(loader, dirname(__file__)))