Try to fix ci failure

python/test/mscclpp_mpi.py

@@ -33,6 +33,11 @@ def finalize_mpi():
         MPI.Finalize()
 
 
+def abort_mpi():
+    if MPI.Is_initialized():
+        MPI.COMM_WORLD.Abort()
+
+
 # Register the function to be called on exit
 atexit.register(finalize_mpi)
 

python/test/test_mscclpp.py

@@ -19,7 +19,7 @@
 )
 from ._cpp import _ext
 from .mscclpp_group import MscclppGroup
-from .mscclpp_mpi import MpiGroup, parametrize_mpi_groups, mpi_group
+from .mscclpp_mpi import MpiGroup, parametrize_mpi_groups, mpi_group, abort_mpi
 from .utils import KernelBuilder, pack
 
 ethernet_interface_name = "eth0"
@@ -205,50 +205,54 @@ def __init__(
         scratch=None,
         fifo=None,
     ):
-        if test_name == "h2d_semaphore":
-            self._kernel = KernelBuilder(
-                file="h2d_semaphore_test.cu",
-                kernel_name="h2d_semaphore",
-            ).get_compiled_kernel()
-            self.nblocks = 1
-            self.nthreads = nranks
-        elif test_name == "d2d_semaphore":
-            self._kernel = KernelBuilder(
-                file="d2d_semaphore_test.cu",
-                kernel_name="d2d_semaphore",
-            ).get_compiled_kernel()
-            self.nblocks = 1
-            self.nthreads = nranks
-        elif test_name == "sm_channel":
-            self._kernel = KernelBuilder(
-                file="sm_channel_test.cu",
-                kernel_name="sm_channel",
-            ).get_compiled_kernel()
-            self.nblocks = nranks
-            self.nthreads = 1024
-        elif test_name == "fifo":
-            self._kernel = KernelBuilder(
-                file="fifo_test.cu",
-                kernel_name="fifo",
-            ).get_compiled_kernel()
-            self.nblocks = 1
-            self.nthreads = 1
-        elif test_name == "proxy":
-            self._kernel = KernelBuilder(
-                file="proxy_test.cu",
-                kernel_name="proxy",
-            ).get_compiled_kernel()
-            self.nblocks = 1
-            self.nthreads = nranks
-        elif test_name == "simple_proxy_channel":
-            self._kernel = KernelBuilder(
-                file="simple_proxy_channel_test.cu",
-                kernel_name="simple_proxy_channel",
-            ).get_compiled_kernel()
-            self.nblocks = 1
-            self.nthreads = 1024
-        else:
-            assert False
+        try:
+            if test_name == "h2d_semaphore":
+                self._kernel = KernelBuilder(
+                    file="h2d_semaphore_test.cu",
+                    kernel_name="h2d_semaphore",
+                ).get_compiled_kernel()
+                self.nblocks = 1
+                self.nthreads = nranks
+            elif test_name == "d2d_semaphore":
+                self._kernel = KernelBuilder(
+                    file="d2d_semaphore_test.cu",
+                    kernel_name="d2d_semaphore",
+                ).get_compiled_kernel()
+                self.nblocks = 1
+                self.nthreads = nranks
+            elif test_name == "sm_channel":
+                self._kernel = KernelBuilder(
+                    file="sm_channel_test.cu",
+                    kernel_name="sm_channel",
+                ).get_compiled_kernel()
+                self.nblocks = nranks
+                self.nthreads = 1024
+            elif test_name == "fifo":
+                self._kernel = KernelBuilder(
+                    file="fifo_test.cu",
+                    kernel_name="fifo",
+                ).get_compiled_kernel()
+                self.nblocks = 1
+                self.nthreads = 1
+            elif test_name == "proxy":
+                self._kernel = KernelBuilder(
+                    file="proxy_test.cu",
+                    kernel_name="proxy",
+                ).get_compiled_kernel()
+                self.nblocks = 1
+                self.nthreads = nranks
+            elif test_name == "simple_proxy_channel":
+                self._kernel = KernelBuilder(
+                    file="simple_proxy_channel_test.cu",
+                    kernel_name="simple_proxy_channel",
+                ).get_compiled_kernel()
+                self.nblocks = 1
+                self.nthreads = 1024
+            else:
+                assert False
+        except Exception:
+            abort_mpi()
+            raise
 
         self.params = b""
         if test_name in ["h2d_semaphore", "d2d_semaphore", "sm_channel", "simple_proxy_channel"]:

python/test/utils.py

@@ -110,7 +110,7 @@ def _compile_cuda(self, source_file, output_file, device_id, std_version="c++17"
         ]
         try:
             subprocess.run(command, capture_output=True, text=True, check=True, bufsize=1)
-            with open(f"{self._tempdir.name}/{output_file}", "rb") as f:
+            with open(os.path.join(self._tempdir.name, output_file), "rb") as f:
                 return f.read()
         except subprocess.CalledProcessError as e:
             raise RuntimeError("Compilation failed:", e.stderr, " ".join(command))

test/deploy/pytest.sh

@@ -5,5 +5,5 @@ if [[ $OMPI_COMM_WORLD_RANK == 0 ]]
 then
   pytest /root/mscclpp/python/test/test_mscclpp.py -x -v
 else
-  pytest /root/mscclpp/python/test/test_mscclpp.py -x 2>&1 >/dev/null
+  pytest /root/mscclpp/python/test/test_mscclpp.py -x >/dev/null
 fi