add parrallel runner for sdc tests

runtests.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+import runpy
+import os
+import sdc
+import numba
+
+
+if __name__ == "__main__":
+    runpy.run_module('sdc.runtests', run_name='__main__')

sdc/runtests.py

@@ -1,68 +1,8 @@
-# *****************************************************************************
-# Copyright (c) 2019-2020, Intel Corporation All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-#     Redistributions of source code must retain the above copyright notice,
-#     this list of conditions and the following disclaimer.
-#
-#     Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions and the following disclaimer in the documentation
-#     and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
-# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
-# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
-# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-# *****************************************************************************
-
-
-import os
-import unittest
-import sdc.tests
-from sdc.tests.test_basic import get_rank
-
-"""
-    Every test in suite can be executed specified times using
-    desired value for SDC_REPEAT_TEST_NUMBER environment variable.
-    This can be used to locate scpecific failures occured
-    on next execution of affected test.
-
-    loadTestsFromModule returns TestSuite obj with _tests member
-    which contains further TestSuite instanses for each found testCase:
-    hpat_tests = TestSuite(sdc.tests)
-    TestSuite(sdc.tests)._tests = [TestSuite(sdc.tests.TestBasic), TestSuite(sdc.tests.TestDataFrame), ...]
-    TestSuite(sdc.tests.TestBasic)._tests = [TestBasic testMethod=test_array_reduce, ...]
-"""
-
-
-def load_tests(loader, tests, pattern):
-    suite = unittest.TestSuite()
-    hpat_tests = loader.loadTestsFromModule(sdc.tests)
-    repeat_test_number = int(os.getenv('SDC_REPEAT_TEST_NUMBER', '1'))
-
-    if repeat_test_number > 1:
-        for i, test_case in enumerate(hpat_tests):
-            extended_tests = []
-            for test in test_case:
-                for _ in range(repeat_test_number):
-                    extended_tests.append(test)
-            hpat_tests._tests[i]._tests = extended_tests
-
-    suite.addTests(hpat_tests)
-    return suite
-
+from sdc.testing._runtests import _main
 
 if __name__ == '__main__':
-    # initialize MPI to avoid "Attempting to use an MPI routine before initializing MPICH" in any pipeline
-    get_rank()
-
-    unittest.main()
+    import sys
+    # For parallel testing under Windows
+    from multiprocessing import freeze_support
+    freeze_support()
+    sys.exit(0 if _main(sys.argv) else 1)

sdc/testing/__init__.py

@@ -0,0 +1,61 @@
+import os
+import sys
+import functools
+import unittest
+import traceback
+from fnmatch import fnmatch
+from os.path import join, isfile, relpath, normpath, splitext, abspath
+import numba
+
+from numba.testing.main import NumbaTestProgram, SerialSuite, make_tag_decorator
+
+
+def load_testsuite(loader, dir):
+    """Find tests in 'dir'."""
+    try:
+        suite = unittest.TestSuite()
+        files = []
+        for f in os.listdir(dir):
+            path = join(dir, f)
+            if isfile(path) and fnmatch(f, 'test_*.py'):
+                files.append(f)
+            elif isfile(join(path, '__init__.py')):
+                suite.addTests(loader.discover(path, top_level_dir=str(abspath(join(path, "../../..")))))
+        for f in files:
+            # turn 'f' into a filename relative to the toplevel dir...
+            f = relpath(join(dir, f), loader._top_level_dir)
+            # ...and translate it to a module name.
+            f = splitext(normpath(f.replace(os.path.sep, '.')))[0]
+            suite.addTests(loader.loadTestsFromName(f))
+        return suite
+    except Exception:
+        traceback.print_exc(file=sys.stderr)
+        sys.exit(-1)
+
+
+def run_tests(argv=None, defaultTest=None, topleveldir=None,
+              xmloutput=None, verbosity=1, nomultiproc=False):
+    """
+    args
+    ----
+    - xmloutput [str or None]
+        Path of XML output directory (optional)
+    - verbosity [int]
+        Verbosity level of tests output
+
+    Returns the TestResult object after running the test *suite*.
+    """
+
+    if xmloutput is not None:
+        import xmlrunner
+        runner = xmlrunner.XMLTestRunner(output=xmloutput)
+    else:
+        runner = None
+    prog = NumbaTestProgram(argv=argv,
+                            module=None,
+                            defaultTest=defaultTest,
+                            topleveldir=topleveldir,
+                            testRunner=runner, exit=False,
+                            verbosity=verbosity,
+                            nomultiproc=nomultiproc)
+    return prog.result

sdc/testing/_runtests.py

@@ -0,0 +1,114 @@
+import json
+import re
+import logging
+
+
+def _main(argv, **kwds):
+    from sdc.testing import run_tests
+    # This helper function assumes the first element of argv
+    # is the name of the calling program.
+    # The 'main' API function is invoked in-process, and thus
+    # will synthesize that name.
+
+    if '--log' in argv:
+        logging.basicConfig(level=logging.DEBUG)
+        argv.remove('--log')
+
+    if '--failed-first' in argv:
+        # Failed first
+        argv.remove('--failed-first')
+        return _FailedFirstRunner().main(argv, kwds)
+    elif '--last-failed' in argv:
+        argv.remove('--last-failed')
+        return _FailedFirstRunner(last_failed=True).main(argv, kwds)
+    else:
+        return run_tests(argv, defaultTest='sdc.tests',
+                         **kwds).wasSuccessful()
+
+
+def main(*argv, **kwds):
+    """keyword arguments are accepted for backward compatibility only.
+    See `numba.testing.run_tests()` documentation for details."""
+    return _main(['<main>'] + list(argv), **kwds)
+
+
+class _FailedFirstRunner(object):
+    """
+    Test Runner to handle the failed-first (--failed-first) option.
+    """
+    cache_filename = '.runtests_lastfailed'
+
+    def __init__(self, last_failed=False):
+        self.last_failed = last_failed
+
+    def main(self, argv, kwds):
+        from sdc.testing import run_tests
+        prog = argv[0]
+        argv = argv[1:]
+        flags = [a for a in argv if a.startswith('-')]
+
+        all_tests, failed_tests = self.find_last_failed(argv)
+        # Prepare tests to run
+        if failed_tests:
+            ft = "There were {} previously failed tests"
+            print(ft.format(len(failed_tests)))
+            remaing_tests = [t for t in all_tests
+                             if t not in failed_tests]
+            if self.last_failed:
+                tests = list(failed_tests)
+            else:
+                tests = failed_tests + remaing_tests
+        else:
+            if self.last_failed:
+                tests = []
+            else:
+                tests = list(all_tests)
+
+        if not tests:
+            print("No tests to run")
+            return True
+        # Run the testsuite
+        print("Running {} tests".format(len(tests)))
+        print('Flags', flags)
+        result = run_tests([prog] + flags + tests, **kwds)
+        # Update failed tests records only if we have run the all the tests
+        # last failed.
+        if len(tests) == result.testsRun:
+            self.save_failed_tests(result, all_tests)
+        return result.wasSuccessful()
+
+    def save_failed_tests(self, result, all_tests):
+        print("Saving failed tests to {}".format(self.cache_filename))
+        cache = []
+        # Find failed tests
+        failed = set()
+        for case in result.errors + result.failures:
+            failed.add(case[0].id())
+        # Build cache
+        for t in all_tests:
+            if t in failed:
+                cache.append(t)
+        # Write cache
+        with open(self.cache_filename, 'w') as fout:
+            json.dump(cache, fout)
+
+    def find_last_failed(self, argv):
+        from numba.tests.support import captured_output
+
+        # Find all tests
+        listargv = ['-l'] + [a for a in argv if not a.startswith('-')]
+        with captured_output("stdout") as stream:
+            main(*listargv)
+
+            pat = re.compile(r"^(\w+\.)+\w+$")
+            lines = stream.getvalue().splitlines()
+        all_tests = [x for x in lines if pat.match(x) is not None]
+
+        try:
+            fobj = open(self.cache_filename)
+        except IOError:
+            failed_tests = []
+        else:
+            with fobj as fin:
+                failed_tests = json.load(fin)
+        return all_tests, failed_tests

sdc/tests/__init__.py

@@ -1,54 +1,10 @@
-# *****************************************************************************
-# Copyright (c) 2019-2020, Intel Corporation All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-#     Redistributions of source code must retain the above copyright notice,
-#     this list of conditions and the following disclaimer.
-#
-#     Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions and the following disclaimer in the documentation
-#     and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
-# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
-# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
-# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-# *****************************************************************************
+from os.path import dirname
+from unittest.suite import TestSuite
+from sdc.testing import load_testsuite
 
 
-from sdc.tests.test_basic import *
-from sdc.tests.test_series import *
-from sdc.tests.test_dataframe import *
-from sdc.tests.test_hiframes import *
-from .categorical import *
-
-# from sdc.tests.test_d4p import *
-from sdc.tests.test_date import *
-from sdc.tests.test_strings import *
-
-from sdc.tests.test_groupby import *
-from sdc.tests.test_join import *
-from sdc.tests.test_rolling import *
-
-from sdc.tests.test_ml import *
-
-from sdc.tests.test_io import *
-
-from sdc.tests.test_hpat_jit import *
-from sdc.tests.test_indexes import *
-
-from sdc.tests.test_sdc_numpy import *
-from sdc.tests.test_prange_utils import *
-from sdc.tests.test_compile_time import *
-
-# performance tests
-import sdc.tests.tests_perf
+def load_tests(loader, tests, pattern):
+    suite = TestSuite()
+    suite.addTests(load_testsuite(loader, dirname(__file__)))
+    return suite
+