From 9d2e812433aa1f1f84b940ab1f74877032c2c4e9 Mon Sep 17 00:00:00 2001 From: viclafargue Date: Tue, 13 Aug 2024 11:25:23 +0200 Subject: [PATCH 1/9] Adding KMeans --- python/cuml/cuml/cluster/kmeans.pyx | 57 +++++++++++-------- .../cuml/cuml/tests/test_device_selection.py | 18 ++++++ 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/python/cuml/cuml/cluster/kmeans.pyx b/python/cuml/cuml/cluster/kmeans.pyx index 760df6306b..e8ab51e4dd 100644 --- a/python/cuml/cuml/cluster/kmeans.pyx +++ b/python/cuml/cuml/cluster/kmeans.pyx @@ -38,7 +38,7 @@ IF GPUBUILD == 1: from cuml.internals.array import CumlArray from cuml.common.array_descriptor import CumlArrayDescriptor -from cuml.internals.base import Base +from cuml.internals.base import UniversalBase from cuml.common.doc_utils import generate_docstring from cuml.internals.mixins import ClusterMixin from cuml.internals.mixins import CMajorInputTagMixin @@ -46,8 +46,10 @@ from cuml.common import input_to_cuml_array from cuml.internals.api_decorators import device_interop_preparation from cuml.internals.api_decorators import enable_device_interop +from sklearn.utils._openmp_helpers import _openmp_effective_n_threads -class KMeans(Base, + +class KMeans(UniversalBase, ClusterMixin, CMajorInputTagMixin): @@ -188,8 +190,8 @@ class KMeans(Base, """ _cpu_estimator_import_path = 'sklearn.cluster.KMeans' - labels_ = CumlArrayDescriptor() - cluster_centers_ = CumlArrayDescriptor() + labels_ = CumlArrayDescriptor(order='C') + cluster_centers_ = CumlArrayDescriptor(order='C') def _get_kmeans_params(self): IF GPUBUILD == 1: @@ -232,6 +234,9 @@ class KMeans(Base, self.labels_ = None self.cluster_centers_ = None + # For sklearn interoperability + self._n_threads = _openmp_effective_n_threads() + # cuPy does not allow comparing with string. See issue #2372 init_str = init if isinstance(init, str) else None @@ -258,7 +263,7 @@ class KMeans(Base, IF GPUBUILD == 1: self._params_init = Array - self.cluster_centers_, _n_rows, self.n_cols, self.dtype = \ + self.cluster_centers_, _n_rows, self.n_features_in_, self.dtype = \ input_to_cuml_array( init, order='C', convert_to_dtype=(np.float32 if convert_dtype @@ -274,7 +279,7 @@ class KMeans(Base, """ if self.init == 'preset': - check_cols = self.n_cols + check_cols = self.n_features_in_ check_dtype = self.dtype target_dtype = self.dtype else: @@ -282,7 +287,7 @@ class KMeans(Base, check_dtype = [np.float32, np.float64] target_dtype = np.float32 - _X_m, _n_rows, self.n_cols, self.dtype = \ + _X_m, _n_rows, self.n_features_in_, self.dtype = \ input_to_cuml_array(X, order='C', check_cols=check_cols, @@ -306,14 +311,14 @@ class KMeans(Base, cdef uintptr_t sample_weight_ptr = sample_weight_m.ptr - int_dtype = np.int32 if np.int64(_n_rows) * np.int64(self.n_cols) < 2**31-1 else np.int64 + int_dtype = np.int32 if np.int64(_n_rows) * np.int64(self.n_features_in_) < 2**31-1 else np.int64 self.labels_ = CumlArray.zeros(shape=_n_rows, dtype=int_dtype) cdef uintptr_t labels_ptr = self.labels_.ptr if (self.init in ['scalable-k-means++', 'k-means||', 'random']): self.cluster_centers_ = \ - CumlArray.zeros(shape=(self.n_clusters, self.n_cols), + CumlArray.zeros(shape=(self.n_clusters, self.n_features_in_), dtype=self.dtype, order='C') cdef uintptr_t cluster_centers_ptr = self.cluster_centers_.ptr @@ -334,7 +339,7 @@ class KMeans(Base, deref(params), input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, sample_weight_ptr, cluster_centers_ptr, labels_ptr, @@ -347,7 +352,7 @@ class KMeans(Base, deref(params), input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, sample_weight_ptr, cluster_centers_ptr, labels_ptr, @@ -364,7 +369,7 @@ class KMeans(Base, deref(params), input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, sample_weight_ptr, cluster_centers_ptr, labels_ptr, @@ -378,7 +383,7 @@ class KMeans(Base, deref(params), input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, sample_weight_ptr, cluster_centers_ptr, labels_ptr, @@ -442,11 +447,13 @@ class KMeans(Base, Sum of squared distances of samples to their closest cluster center. """ + self.dtype = self.cluster_centers_.dtype + _X_m, _n_rows, _n_cols, _ = \ input_to_cuml_array(X, order='C', check_dtype=self.dtype, convert_to_dtype=(self.dtype if convert_dtype else None), - check_cols=self.n_cols) + check_cols=self.n_features_in_) IF GPUBUILD == 1: cdef uintptr_t input_ptr = _X_m.ptr @@ -486,7 +493,7 @@ class KMeans(Base, cluster_centers_ptr, input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, sample_weight_ptr, normalize_weights, labels_ptr, @@ -498,7 +505,7 @@ class KMeans(Base, cluster_centers_ptr, input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, sample_weight_ptr, normalize_weights, labels_ptr, @@ -513,7 +520,7 @@ class KMeans(Base, cluster_centers_ptr, input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, sample_weight_ptr, normalize_weights, labels_ptr, @@ -525,7 +532,7 @@ class KMeans(Base, cluster_centers_ptr, input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, sample_weight_ptr, normalize_weights, labels_ptr, @@ -578,7 +585,7 @@ class KMeans(Base, input_to_cuml_array(X, order='C', check_dtype=self.dtype, convert_to_dtype=(self.dtype if convert_dtype else None), - check_cols=self.n_cols) + check_cols=self.n_features_in_) IF GPUBUILD == 1: cdef uintptr_t input_ptr = _X_m.ptr @@ -607,7 +614,7 @@ class KMeans(Base, cluster_centers_ptr, input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, preds_ptr) else: cpp_transform( @@ -616,7 +623,7 @@ class KMeans(Base, cluster_centers_ptr, input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, preds_ptr) elif self.dtype == np.float64: @@ -627,7 +634,7 @@ class KMeans(Base, cluster_centers_ptr, input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, preds_ptr) else: cpp_transform( @@ -636,7 +643,7 @@ class KMeans(Base, cluster_centers_ptr, input_ptr, _n_rows, - self.n_cols, + self.n_features_in_, preds_ptr) else: @@ -685,3 +692,7 @@ class KMeans(Base, ['n_init', 'oversampling_factor', 'max_samples_per_batch', 'init', 'max_iter', 'n_clusters', 'random_state', 'tol', "convert_dtype"] + + def get_attr_names(self): + return ['cluster_centers_', 'labels_', 'inertia_', + 'n_iter_', 'n_features_in_', '_n_threads'] diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index e5c2d9ce1a..57f8965674 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -32,6 +32,7 @@ from cuml.internals.memory_utils import using_memory_type from cuml.internals.mem_type import MemoryType from cuml.decomposition import PCA, TruncatedSVD +from cuml.cluster import KMeans from cuml.common.device_selection import DeviceType, using_device_type from hdbscan import HDBSCAN as refHDBSCAN from sklearn.neighbors import NearestNeighbors as skNearestNeighbors @@ -42,6 +43,7 @@ from sklearn.linear_model import LinearRegression as skLinearRegression from sklearn.decomposition import PCA as skPCA from sklearn.decomposition import TruncatedSVD as skTruncatedSVD +from sklearn.cluster import KMeans as skKMeans from sklearn.datasets import make_regression, make_blobs from pytest_cases import fixture_union, fixture from importlib import import_module @@ -948,3 +950,19 @@ def test_hdbscan_methods(train_device, infer_device): assert_membership_vectors(membership, ref_membership) assert adjusted_rand_score(labels, ref_labels) >= 0.98 assert array_equal(probs, ref_probs, unit_tol=0.001, total_tol=0.006) + + +@pytest.mark.parametrize("train_device", ["cpu", "gpu"]) +@pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) +def test_kmeans_methods(train_device, infer_device): + ref_model = skKMeans(n_clusters=20) + ref_model.fit(X_train_blob) + ref_output = ref_model.predict(X_train_blob) + + model = KMeans(n_clusters=20) + with using_device_type(train_device): + model.fit(X_train_blob) + with using_device_type(infer_device): + output = model.predict(X_train_blob) + + assert adjusted_rand_score(ref_output, output) >= 0.95 From fff6ebe1b8d6ec79fe9372ccf29c336e0e58a27f Mon Sep 17 00:00:00 2001 From: viclafargue Date: Tue, 13 Aug 2024 14:17:00 +0200 Subject: [PATCH 2/9] Adding DBSCAN --- python/cuml/cuml/cluster/dbscan.pyx | 21 ++++++---- .../cuml/cuml/tests/test_device_selection.py | 41 ++++++++++++++++--- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/python/cuml/cuml/cluster/dbscan.pyx b/python/cuml/cuml/cluster/dbscan.pyx index b1a8dd5ae8..fff1eef3f9 100644 --- a/python/cuml/cuml/cluster/dbscan.pyx +++ b/python/cuml/cuml/cluster/dbscan.pyx @@ -22,7 +22,7 @@ from cuml.internals.safe_imports import gpu_only_import cp = gpu_only_import('cupy') from cuml.internals.array import CumlArray -from cuml.internals.base import Base +from cuml.internals.base import UniversalBase from cuml.common.doc_utils import generate_docstring from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals.mixins import ClusterMixin @@ -106,7 +106,7 @@ IF GPUBUILD == 1: bool opg) except + -class DBSCAN(Base, +class DBSCAN(UniversalBase, ClusterMixin, CMajorInputTagMixin): """ @@ -222,8 +222,8 @@ class DBSCAN(Base, """ _cpu_estimator_import_path = 'sklearn.cluster.DBSCAN' - labels_ = CumlArrayDescriptor() - core_sample_indices_ = CumlArrayDescriptor() + core_sample_indices_ = CumlArrayDescriptor(order="C") + labels_ = CumlArrayDescriptor(order="C") @device_interop_preparation def __init__(self, *, @@ -268,7 +268,7 @@ class DBSCAN(Base, "np.int32, np.int64}") IF GPUBUILD == 1: - X_m, n_rows, n_cols, self.dtype = \ + X_m, n_rows, self.n_features_in_, self.dtype = \ input_to_cuml_array( X, order='C', @@ -338,7 +338,7 @@ class DBSCAN(Base, fit(handle_[0], input_ptr, n_rows, - n_cols, + self.n_features_in_, self.eps, self.min_samples, metric, @@ -353,7 +353,7 @@ class DBSCAN(Base, fit(handle_[0], input_ptr, n_rows, - n_cols, + self.n_features_in_, self.eps, self.min_samples, metric, @@ -370,7 +370,7 @@ class DBSCAN(Base, fit(handle_[0], input_ptr, n_rows, - n_cols, + self.n_features_in_, self.eps, self.min_samples, metric, @@ -385,7 +385,7 @@ class DBSCAN(Base, fit(handle_[0], input_ptr, n_rows, - n_cols, + self.n_features_in_, self.eps, self.min_samples, metric, @@ -475,3 +475,6 @@ class DBSCAN(Base, "metric", "algorithm", ] + + def get_attr_names(self): + return ["core_sample_indices_", "labels_", "n_features_in_"] diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index 57f8965674..1da3b0738e 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -33,7 +33,9 @@ from cuml.internals.mem_type import MemoryType from cuml.decomposition import PCA, TruncatedSVD from cuml.cluster import KMeans +from cuml.cluster import DBSCAN from cuml.common.device_selection import DeviceType, using_device_type +from cuml.testing.utils import assert_dbscan_equal from hdbscan import HDBSCAN as refHDBSCAN from sklearn.neighbors import NearestNeighbors as skNearestNeighbors from sklearn.linear_model import Ridge as skRidge @@ -44,6 +46,7 @@ from sklearn.decomposition import PCA as skPCA from sklearn.decomposition import TruncatedSVD as skTruncatedSVD from sklearn.cluster import KMeans as skKMeans +from sklearn.cluster import DBSCAN as skDBSCAN from sklearn.datasets import make_regression, make_blobs from pytest_cases import fixture_union, fixture from importlib import import_module @@ -138,7 +141,11 @@ def make_reg_dataset(): def make_blob_dataset(): X, y = make_blobs( - n_samples=2000, n_features=20, centers=20, random_state=0 + n_samples=2000, + n_features=20, + centers=20, + random_state=0, + cluster_std=1.0, ) X_train, X_test = X[:1800], X[1800:] y_train, _ = y[:1800], y[1800:] @@ -955,14 +962,38 @@ def test_hdbscan_methods(train_device, infer_device): @pytest.mark.parametrize("train_device", ["cpu", "gpu"]) @pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) def test_kmeans_methods(train_device, infer_device): - ref_model = skKMeans(n_clusters=20) + n_clusters = 20 + ref_model = skKMeans(n_clusters=n_clusters) ref_model.fit(X_train_blob) - ref_output = ref_model.predict(X_train_blob) + ref_output = ref_model.predict(X_test_blob) - model = KMeans(n_clusters=20) + model = KMeans(n_clusters=n_clusters) with using_device_type(train_device): model.fit(X_train_blob) with using_device_type(infer_device): - output = model.predict(X_train_blob) + output = model.predict(X_test_blob) + assert adjusted_rand_score(ref_output, output) >= 0.9 + + +@pytest.mark.parametrize("train_device", ["cpu", "gpu"]) +@pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) +def test_dbscan_methods(train_device, infer_device): + eps = 8.0 + ref_model = skDBSCAN(eps=eps) + ref_model.fit(X_train_blob) + ref_output = ref_model.fit_predict(X_train_blob) + + model = DBSCAN(eps=eps) + with using_device_type(train_device): + model.fit(X_train_blob) + with using_device_type(infer_device): + output = model.fit_predict(X_train_blob) + + assert array_equal( + ref_model.core_sample_indices_, ref_model.core_sample_indices_ + ) assert adjusted_rand_score(ref_output, output) >= 0.95 + assert_dbscan_equal( + ref_output, output, X_train_blob, model.core_sample_indices_, eps + ) From 712a294b028edc22a1a31e444bcf8821496e3c2c Mon Sep 17 00:00:00 2001 From: viclafargue Date: Mon, 30 Sep 2024 18:04:06 +0200 Subject: [PATCH 3/9] SVM CPU/GPU interop --- python/cuml/cuml/multiclass/multiclass.py | 29 ++++ python/cuml/cuml/svm/svc.pyx | 86 +++++++-- python/cuml/cuml/svm/svm_base.pyx | 164 ++++++++++++------ python/cuml/cuml/svm/svr.pyx | 25 ++- .../cuml/cuml/tests/test_device_selection.py | 55 +++++- 5 files changed, 292 insertions(+), 67 deletions(-) diff --git a/python/cuml/cuml/multiclass/multiclass.py b/python/cuml/cuml/multiclass/multiclass.py index 58c4151094..7ae123b2ff 100644 --- a/python/cuml/cuml/multiclass/multiclass.py +++ b/python/cuml/cuml/multiclass/multiclass.py @@ -118,6 +118,35 @@ def __init__( def classes_(self): return self.multiclass_estimator.classes_ + @classes_.setter + def classes_(self, value): + import sklearn.multiclass + + if self.strategy == "ovr": + self.multiclass_estimator = sklearn.multiclass.OneVsRestClassifier( + self.estimator, n_jobs=None + ) + from sklearn.preprocessing import LabelBinarizer + + self.multiclass_estimator.label_binarizer_ = LabelBinarizer( + sparse_output=True + ) + self.multiclass_estimator.label_binarizer_.fit( + value.to_output("numpy") + ) + elif self.strategy == "ovo": + self.multiclass_estimator = sklearn.multiclass.OneVsOneClassifier( + self.estimator, n_jobs=None + ) + else: + raise ValueError( + "Invalid multiclass strategy " + + str(self.strategy) + + ", must be one of " + '{"ovr", "ovo"}' + ) + self.multiclass_estimator.classes_ = value + @property @cuml.internals.api_base_return_any_skipall def n_classes_(self): diff --git a/python/cuml/cuml/svm/svc.pyx b/python/cuml/cuml/svm/svc.pyx index 566aa4b762..7b5781605d 100644 --- a/python/cuml/cuml/svm/svc.pyx +++ b/python/cuml/cuml/svm/svc.pyx @@ -28,10 +28,12 @@ cuda = gpu_only_import_from('numba', 'cuda') from cython.operator cimport dereference as deref from libc.stdint cimport uintptr_t +import warnings import cuml.internals from cuml.internals.array import CumlArray from cuml.internals.mixins import ClassifierMixin from cuml.common.doc_utils import generate_docstring +from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals.logger import warn from pylibraft.common.handle cimport handle_t from pylibraft.common.interruptible import cuda_interruptible @@ -42,6 +44,7 @@ from libcpp cimport nullptr from cuml.svm.svm_base import SVMBase from cuml.internals.import_utils import has_sklearn from cuml.internals.array_sparse import SparseCumlArray +from cuml.internals.api_decorators import device_interop_preparation, enable_device_interop if has_sklearn(): from cuml.multiclass import MulticlassClassifier @@ -259,7 +262,7 @@ class SVC(SVMBase, max_iter : int (default = -1) Limit the number of outer iterations in the solver. If -1 (default) then ``max_iter=100*n_samples`` - multiclass_strategy : str ('ovo' or 'ovr', default 'ovo') + decision_function_shape : str ('ovo' or 'ovr', default 'ovo') Multiclass classification strategy. ``'ovo'`` uses `OneVsOneClassifier `_ while ``'ovr'`` selects `OneVsRestClassifier @@ -330,11 +333,17 @@ class SVC(SVMBase, """ + _cpu_estimator_import_path = 'sklearn.svm.SVC' + + class_weight_ = CumlArrayDescriptor(order='F') + + @device_interop_preparation def __init__(self, *, handle=None, C=1, kernel='rbf', degree=3, gamma='scale', coef0=0.0, tol=1e-3, cache_size=1024.0, max_iter=-1, nochange_steps=1000, verbose=False, output_type=None, probability=False, random_state=None, - class_weight=None, multiclass_strategy='ovo'): + class_weight=None, decision_function_shape='ovo', + multiclass_strategy=None): super().__init__( handle=handle, C=C, @@ -355,7 +364,16 @@ class SVC(SVMBase, warn("Random state is currently ignored by probabilistic SVC") self.class_weight = class_weight self.svmType = C_SVC - self.multiclass_strategy = multiclass_strategy + + if multiclass_strategy: + decision_function_shape = multiclass_strategy + warnings.simplefilter(action="always", category=FutureWarning) + warnings.warn('Parameter "multiclass_strategy" has been' + ' deprecated. Please use the' + ' "decision_function_shape" parameter instead.', + FutureWarning) + + self.decision_function_shape = decision_function_shape @property @cuml.internals.api_base_return_array_skipall @@ -367,6 +385,15 @@ class SVC(SVMBase, else: return self._unique_labels_ + @classes_.setter + def classes_(self, value): + if self.probability: + self.prob_svc.classes_ = value + elif self.n_classes_ > 2: + self.multiclass_svc.classes_ = value + else: + self._unique_labels_ = CumlArray.from_input(value, convert_to_dtype=self.dtype) + @property @cuml.internals.api_base_return_array_skipall def support_(self): @@ -410,7 +437,7 @@ class SVC(SVMBase, raise RuntimeError("Scikit-learn is needed to fit multiclass SVM") params = self.get_params() - strategy = params.pop('multiclass_strategy', 'ovo') + strategy = params.pop('decision_function_shape', 'ovo') self.multiclass_svc = MulticlassClassifier( estimator=SVC(**params), handle=self.handle, verbose=self.verbose, @@ -437,7 +464,7 @@ class SVC(SVMBase, ].support_ = cp.nonzero(cond)[0][ovo_support] estimator_index += 1 - self._fit_status_ = 0 + self.fit_status_ = 0 return self def _fit_proba(self, X, y, sample_weight) -> "SVC": @@ -478,11 +505,12 @@ class SVC(SVMBase, # Fit the model, sample_weight is either None or a numpy array self.prob_svc.fit(X, y, sample_weight=sample_weight) - self._fit_status_ = 0 + self.fit_status_ = 0 return self @generate_docstring(y='dense_anydtype') @cuml.internals.api_base_return_any(set_output_dtype=True) + @enable_device_interop def fit(self, X, y, sample_weight=None, convert_dtype=True) -> "SVC": """ Fit the model with X and y. @@ -494,6 +522,11 @@ class SVC(SVMBase, # we need to check whether input X is sparse # In that case we don't want to make a dense copy _array_type, is_sparse = determine_array_type_full(X) + self._sparse = is_sparse + + # n = int(self.n_classes_ * (self.n_classes_ - 1) / 2) + # self._probA = np.empty(n, dtype=np.float64) + # self._probB = np.empty(n, dtype=np.float64) if self.probability: return self._fit_proba(X, y, sample_weight) @@ -504,10 +537,10 @@ class SVC(SVMBase, if is_sparse: X_m = SparseCumlArray(X) self.n_rows = X_m.shape[0] - self.n_cols = X_m.shape[1] + self.n_features_in_ = X_m.shape[1] self.dtype = X_m.dtype else: - X_m, self.n_rows, self.n_cols, self.dtype = \ + X_m, self.n_rows, self.n_features_in_, self.dtype = \ input_to_cuml_array(X, order='F') # Fit binary classifier @@ -538,7 +571,7 @@ class SVC(SVMBase, cdef handle_t* handle_ = self.handle.getHandle() cdef int n_rows = self.n_rows - cdef int n_cols = self.n_cols + cdef int n_cols = self.n_features_in_ cdef int n_nnz = X_m.nnz if is_sparse else -1 cdef uintptr_t X_indptr = X_m.indptr.ptr if is_sparse else X_m.ptr @@ -585,7 +618,7 @@ class SVC(SVMBase, raise TypeError('Input data type should be float32 or float64') self._unpack_model() - self._fit_status_ = 0 + self.fit_status_ = 0 self.handle.sync() del X_m @@ -597,6 +630,7 @@ class SVC(SVMBase, 'type': 'dense', 'description': 'Predicted values', 'shape': '(n_samples, 1)'}) + @enable_device_interop def predict(self, X, convert_dtype=True) -> CumlArray: """ Predicts the class labels for X. The returned y values are the class @@ -624,6 +658,7 @@ class SVC(SVMBase, 'description': 'Predicted \ probabilities', 'shape': '(n_samples, n_classes)'}) + @enable_device_interop def predict_proba(self, X, log=False) -> CumlArray: """ Predicts the class probabilities for X. @@ -661,6 +696,7 @@ class SVC(SVMBase, probabilities', 'shape': '(n_samples, n_classes)'}) @cuml.internals.api_base_return_array_skipall + @enable_device_interop def predict_log_proba(self, X) -> CumlArray: """ Predicts the log probabilities for X (returns log(predict_proba(x)). @@ -675,6 +711,7 @@ class SVC(SVMBase, 'description': 'Decision function \ values', 'shape': '(n_samples, 1)'}) + @enable_device_interop def decision_function(self, X) -> CumlArray: """ Calculates the decision function values for X. @@ -704,11 +741,36 @@ class SVC(SVMBase, def get_param_names(self): params = super().get_param_names() + \ - ["probability", "random_state", "class_weight", - "multiclass_strategy"] + ["probability", "random_state", "class_weight", "decision_function_shape"] # Ignore "epsilon" since its not used in the constructor if ("epsilon" in params): params.remove("epsilon") return params + + def get_attr_names(self): + return super().get_attr_names() + ["classes_", "_sparse"] + + def cpu_to_gpu(self): + self.dtype = np.float64 + self.probability = self._cpu_model.probability + self.n_classes_ = len(self._cpu_model.classes_) + + if self.probability: + params = self.get_params() + params["probability"] = False + params["output_type"] = "numpy" + self.prob_svc = CalibratedClassifierCV(SVC(**params), cv=5, method='sigmoid') + elif self.n_classes_ > 2: + if not hasattr(self, 'multiclass_svc'): + params = self.get_params() + strategy = params.pop('decision_function_shape', 'ovo') + self.multiclass_svc = \ + MulticlassClassifier(estimator=SVC(**params), handle=self.handle, + verbose=self.verbose, output_type=self.output_type, + strategy=strategy) + else: + super().cpu_to_gpu() + self.n_support_ = self._cpu_model.n_support_ + self._model = self._get_svm_model() diff --git a/python/cuml/cuml/svm/svm_base.pyx b/python/cuml/cuml/svm/svm_base.pyx index 1a478fbc9c..d8702bb861 100644 --- a/python/cuml/cuml/svm/svm_base.pyx +++ b/python/cuml/cuml/svm/svm_base.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ from libc.stdint cimport uintptr_t import cuml.internals from cuml.internals.array import CumlArray from cuml.common.array_descriptor import CumlArrayDescriptor -from cuml.internals.base import Base +from cuml.internals.base import UniversalBase from cuml.common.exceptions import NotFittedError from pylibraft.common.handle cimport handle_t from cuml.common import input_to_cuml_array @@ -109,7 +109,7 @@ cdef extern from "cuml/svm/svc.hpp" namespace "ML::SVM": SvmModel[math_t] &m) except + -class SVMBase(Base, +class SVMBase(UniversalBase, FMajorInputTagMixin): """ Base class for Support Vector Machines @@ -214,12 +214,12 @@ class SVMBase(Base, """ - dual_coef_ = CumlArrayDescriptor() - support_ = CumlArrayDescriptor() - support_vectors_ = CumlArrayDescriptor() - _intercept_ = CumlArrayDescriptor() - _internal_coef_ = CumlArrayDescriptor() - _unique_labels_ = CumlArrayDescriptor() + dual_coef_ = CumlArrayDescriptor(order='F') + support_ = CumlArrayDescriptor(order='F') + support_vectors__ = CumlArrayDescriptor(order='F') + _intercept__ = CumlArrayDescriptor(order='F') + _internal_coef_ = CumlArrayDescriptor(order='F') + _unique_labels_ = CumlArrayDescriptor(order='F') def __init__(self, *, handle=None, C=1, kernel='rbf', degree=3, gamma='auto', coef0=0.0, tol=1e-3, cache_size=1024.0, @@ -243,17 +243,17 @@ class SVMBase(Base, # Parameter to indicate if model has been correctly fitted # fit_status == -1 indicates that the model is not yet fitted - self._fit_status_ = -1 + self.fit_status_ = -1 # Attributes (parameters of the fitted model) self.dual_coef_ = None self.support_ = None - self.support_vectors_ = None - self._intercept_ = None + self.support_vectors__ = None + self._intercept__ = None self.n_support_ = None self._c_kernel = self._get_c_kernel(kernel) - self._gamma_val = None # the actual numerical value used for training + self._gamma = None # the actual numerical value used for training self.coef_ = None # value of the coef_ attribute, only for lin kernel self.dtype = None self._model = None # structure of the model parameters @@ -289,7 +289,7 @@ class SVMBase(Base, else: raise TypeError("Unknown type for SVC class") try: - del self._fit_status_ + del self.fit_status_ except AttributeError: pass @@ -322,28 +322,28 @@ class SVMBase(Base, """ if type(self.gamma) is str: if self.gamma == 'auto': - return 1 / self.n_cols + return 1 / self.n_features_in_ elif self.gamma == 'scale': if isinstance(X, SparseCumlArray): # account for zero values data_cupy = cupy.asarray(X.data).copy() - num_elements = self.n_cols * self.n_rows + num_elements = self.n_features_in_ * self.n_rows extended_mean = data_cupy.mean()*X.nnz/num_elements data_cupy = (data_cupy - extended_mean)**2 x_var = (data_cupy.sum() + (num_elements-X.nnz)*extended_mean*extended_mean)/num_elements else: x_var = cupy.asarray(X).var().item() - return 1 / (self.n_cols * x_var) + return 1 / (self.n_features_in_ * x_var) else: raise ValueError("Not implemented gamma option: " + self.gamma) else: return self.gamma def _calc_coef(self): - if (self.n_support_ == 0): - return cupy.zeros((1, self.n_cols), dtype=self.dtype) + if self.n_support_ == 0: + return cupy.zeros((1, self.n_features_in_), dtype=self.dtype) with using_output_type("cupy"): - return cupy.dot(self.dual_coef_, self.support_vectors_) + return cupy.dot(self.dual_coef_, self.support_vectors__) def _check_is_fitted(self, attr): if not hasattr(self, attr) or (getattr(self, attr) is None): @@ -367,25 +367,79 @@ class SVMBase(Base, def coef_(self, value): self._internal_coef_ = value + @property + def _probA(self): + if not hasattr(self, '__probA'): + return np.empty(0, dtype=np.float64) + else: + return self.__probA + + @_probA.setter + def _probA(self, value): + self.__probA = value + + @property + def _probB(self): + if not hasattr(self, '__probB'): + return np.empty(0, dtype=np.float64) + else: + return self.__probB + + @_probB.setter + def _probB(self, value): + self.__probB = value + @property @cuml.internals.api_base_return_array_skipall def intercept_(self): - if self._intercept_ is None: + if self._intercept__ is None: raise AttributeError("intercept_ called before fit.") - return self._intercept_ + return self._intercept__ @intercept_.setter def intercept_(self, value): - self._intercept_ = value + self._intercept__ = value + + @property + @cuml.internals.api_base_return_array_skipall + def _intercept_(self): + if self._intercept__ is None: + raise AttributeError("intercept_ called before fit.") + return self._intercept__.to_output('numpy').astype(np.float64) + + @_intercept_.setter + def _intercept_(self, value): + if isinstance(value, CumlArray): + value = value.to_output('cupy') + self._intercept__ = cupy.ascontiguousarray(value) + + @property + def _dual_coef_(self): + return self.dual_coef_.to_output('numpy').astype(np.float64) + + @_dual_coef_.setter + def _dual_coef_(self, value): + if isinstance(value, CumlArray): + value = value.to_output('cupy') + self.dual_coef_ = cupy.ascontiguousarray(value) + + @property + def support_vectors_(self): + support_vectors = self.support_vectors__.to_output('numpy').astype(np.float64) + return np.ascontiguousarray(support_vectors) + + @support_vectors_.setter + def support_vectors_(self, value): + self.support_vectors__ = value def _get_kernel_params(self, X=None): """ Wrap the kernel parameters in a KernelParams obtect """ cdef KernelParams _kernel_params if X is not None: - self._gamma_val = self._calc_gamma_val(X) + self._gamma = self._calc_gamma_val(X) _kernel_params.kernel = self._c_kernel _kernel_params.degree = self.degree - _kernel_params.gamma = self._gamma_val + _kernel_params.gamma = self._gamma _kernel_params.coef0 = self.coef0 return _kernel_params @@ -413,20 +467,23 @@ class SVMBase(Base, if self.dual_coef_ is None: # the model is not fitted in this case return None + + n_support = self.n_support_.sum() + if self.dtype == np.float32: model_f = new SvmModel[float]() - model_f.n_support = self.n_support_ - model_f.n_cols = self.n_cols - model_f.b = self._intercept_.item() + model_f.n_support = n_support + model_f.n_cols = self.n_features_in_ + model_f.b = self._intercept__.item() model_f.dual_coefs = \ self.dual_coef_.ptr - if isinstance(self.support_vectors_, SparseCumlArray): - model_f.support_matrix.nnz = self.support_vectors_.nnz - model_f.support_matrix.indptr = self.support_vectors_.indptr.ptr - model_f.support_matrix.indices = self.support_vectors_.indices.ptr - model_f.support_matrix.data = self.support_vectors_.data.ptr + if isinstance(self.support_vectors__, SparseCumlArray): + model_f.support_matrix.nnz = self.support_vectors__.nnz + model_f.support_matrix.indptr = self.support_vectors__.indptr.ptr + model_f.support_matrix.indices = self.support_vectors__.indices.ptr + model_f.support_matrix.data = self.support_vectors__.data.ptr else: - model_f.support_matrix.data = self.support_vectors_.ptr + model_f.support_matrix.data = self.support_vectors__.ptr model_f.support_idx = \ self.support_.ptr model_f.n_classes = self.n_classes_ @@ -438,18 +495,18 @@ class SVMBase(Base, return model_f else: model_d = new SvmModel[double]() - model_d.n_support = self.n_support_ - model_d.n_cols = self.n_cols - model_d.b = self._intercept_.item() + model_d.n_support = n_support + model_d.n_cols = self.n_features_in_ + model_d.b = self._intercept__.item() model_d.dual_coefs = \ self.dual_coef_.ptr - if isinstance(self.support_vectors_, SparseCumlArray): - model_d.support_matrix.nnz = self.support_vectors_.nnz - model_d.support_matrix.indptr = self.support_vectors_.indptr.ptr - model_d.support_matrix.indices = self.support_vectors_.indices.ptr - model_d.support_matrix.data = self.support_vectors_.data.ptr + if isinstance(self.support_vectors__, SparseCumlArray): + model_d.support_matrix.nnz = self.support_vectors__.nnz + model_d.support_matrix.indptr = self.support_vectors__.indptr.ptr + model_d.support_matrix.indices = self.support_vectors__.indices.ptr + model_d.support_matrix.data = self.support_vectors__.data.ptr else: - model_d.support_matrix.data = self.support_vectors_.ptr + model_d.support_matrix.data = self.support_vectors__.ptr model_d.support_idx = \ self.support_.ptr model_d.n_classes = self.n_classes_ @@ -461,7 +518,7 @@ class SVMBase(Base, return model_d def _unpack_svm_model(self, b, n_support, dual_coefs, support_idx, nnz, indptr, indices, data, n_classes, unique_labels): - self._intercept_ = CumlArray.full(1, b, self.dtype) + self._intercept__ = CumlArray.full(1, b, self.dtype) self.n_support_ = n_support if n_support > 0: @@ -478,9 +535,9 @@ class SVMBase(Base, order='F') if nnz == -1: - self.support_vectors_ = CumlArray( + self.support_vectors__ = CumlArray( data=data, - shape=(self.n_support_, self.n_cols), + shape=(self.n_support_, self.n_features_in_), dtype=self.dtype, order='F') else: @@ -502,8 +559,8 @@ class SVMBase(Base, indices=indices, data=data, nnz=nnz, - shape=(self.n_support_, self.n_cols)) - self.support_vectors_ = SparseCumlArray(data=sparse_input) + shape=(self.n_support_, self.n_features_in_)) + self.support_vectors__ = SparseCumlArray(data=sparse_input) self.n_classes_ = n_classes if self.n_classes_ > 0: @@ -565,7 +622,7 @@ class SVMBase(Base, # Setting all dims to zero due to issue # https://github.com/rapidsai/cuml/issues/4095 - self.support_vectors_ = CumlArray.empty( + self.support_vectors__ = CumlArray.empty( shape=(0, 0), dtype=self.dtype, order='F') @@ -675,6 +732,15 @@ class SVMBase(Base, "epsilon", ] + def get_attr_names(self): + attr_names = ["_dual_coef_", "fit_status_", "_intercept_", + "n_features_in_", "_n_support", "shape_fit_", + "support_", "support_vectors_", + "_probA", "_probB", "_gamma"] + if self.kernel == "linear": + attr_names.append("coef_") + return attr_names + def __getstate__(self): state = self.__dict__.copy() del state['handle'] diff --git a/python/cuml/cuml/svm/svr.pyx b/python/cuml/cuml/svm/svr.pyx index a2527f4358..b308774653 100644 --- a/python/cuml/cuml/svm/svr.pyx +++ b/python/cuml/cuml/svm/svr.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.input_utils import determine_array_type_full from cuml.internals.mixins import RegressorMixin +from cuml.internals.api_decorators import device_interop_preparation, enable_device_interop from cuml.common.doc_utils import generate_docstring from pylibraft.common.handle cimport handle_t from cuml.common import input_to_cuml_array @@ -227,6 +228,10 @@ class SVR(SVMBase, RegressorMixin): Predicted values: [1.200474 3.8999617 5.100488 3.7995374 1.0995375] """ + + _cpu_estimator_import_path = 'sklearn.svm.SVR' + + @device_interop_preparation def __init__(self, *, handle=None, C=1, kernel='rbf', degree=3, gamma='scale', coef0=0.0, tol=1e-3, epsilon=0.1, cache_size=1024.0, max_iter=-1, nochange_steps=1000, @@ -250,6 +255,7 @@ class SVR(SVMBase, RegressorMixin): self.svmType = EPSILON_SVR @generate_docstring() + @enable_device_interop def fit(self, X, y, sample_weight=None, convert_dtype=True) -> "SVR": """ Fit the model with X and y. @@ -258,14 +264,15 @@ class SVR(SVMBase, RegressorMixin): # we need to check whether out input X is sparse # In that case we don't want to make a dense copy _array_type, is_sparse = determine_array_type_full(X) + self._sparse = is_sparse if is_sparse: X_m = SparseCumlArray(X) self.n_rows = X_m.shape[0] - self.n_cols = X_m.shape[1] + self.n_features_in_ = X_m.shape[1] self.dtype = X_m.dtype else: - X_m, self.n_rows, self.n_cols, self.dtype = \ + X_m, self.n_rows, self.n_features_in_, self.dtype = \ input_to_cuml_array(X, order='F') convert_to_dtype = self.dtype if convert_dtype else None @@ -294,7 +301,7 @@ class SVR(SVMBase, RegressorMixin): cdef handle_t* handle_ = self.handle.getHandle() cdef int n_rows = self.n_rows - cdef int n_cols = self.n_cols + cdef int n_cols = self.n_features_in_ cdef int n_nnz = X_m.nnz if is_sparse else -1 cdef uintptr_t X_indptr = X_m.indptr.ptr if is_sparse else X_m.ptr cdef uintptr_t X_indices = X_m.indices.ptr if is_sparse else X_m.ptr @@ -328,7 +335,7 @@ class SVR(SVMBase, RegressorMixin): raise TypeError('Input data type should be float32 or float64') self._unpack_model() - self._fit_status_ = 0 + self.fit_status_ = 0 self.handle.sync() del X_m @@ -340,6 +347,7 @@ class SVR(SVMBase, RegressorMixin): 'type': 'dense', 'description': 'Predicted values', 'shape': '(n_samples, 1)'}) + @enable_device_interop def predict(self, X, convert_dtype=True) -> CumlArray: """ Predicts the values for X. @@ -347,3 +355,10 @@ class SVR(SVMBase, RegressorMixin): """ return super(SVR, self).predict(X, False, convert_dtype) + + def get_attr_names(self): + return super().get_attr_names() + ["_sparse"] + + def gpu_to_cpu(self): + super().gpu_to_cpu() + self._cpu_model._n_support = np.array([self.n_support_], dtype=np.int32) diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index 1da3b0738e..80359d9298 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -34,6 +34,7 @@ from cuml.decomposition import PCA, TruncatedSVD from cuml.cluster import KMeans from cuml.cluster import DBSCAN +from cuml.svm import SVC, SVR from cuml.common.device_selection import DeviceType, using_device_type from cuml.testing.utils import assert_dbscan_equal from hdbscan import HDBSCAN as refHDBSCAN @@ -47,7 +48,9 @@ from sklearn.decomposition import TruncatedSVD as skTruncatedSVD from sklearn.cluster import KMeans as skKMeans from sklearn.cluster import DBSCAN as skDBSCAN -from sklearn.datasets import make_regression, make_blobs +from sklearn.svm import SVC as skSVC +from sklearn.svm import SVR as skSVR +from sklearn.datasets import make_regression, make_classification, make_blobs from pytest_cases import fixture_union, fixture from importlib import import_module import inspect @@ -139,6 +142,23 @@ def make_reg_dataset(): ) +def make_class_dataset(): + X, y = make_classification( + n_samples=2000, + n_features=20, + n_informative=18, + n_classes=2, + random_state=0, + ) + X_train, X_test = X[:1800], X[1800:] + y_train, _ = y[:1800], y[1800:] + return ( + X_train.astype(np.float32), + y_train.astype(np.float32), + X_test.astype(np.float32), + ) + + def make_blob_dataset(): X, y = make_blobs( n_samples=2000, @@ -157,6 +177,7 @@ def make_blob_dataset(): X_train_reg, y_train_reg, X_test_reg = make_reg_dataset() +X_train_class, y_train_class, X_test_class = make_class_dataset() X_train_blob, y_train_blob, X_test_blob = make_blob_dataset() @@ -997,3 +1018,35 @@ def test_dbscan_methods(train_device, infer_device): assert_dbscan_equal( ref_output, output, X_train_blob, model.core_sample_indices_, eps ) + + +@pytest.mark.parametrize("train_device", ["cpu", "gpu"]) +@pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) +def test_svc_methods(train_device, infer_device): + ref_model = skSVC() + ref_model.fit(X_train_class, y_train_class) + ref_output = ref_model.predict(X_test_class) + + model = SVC() + with using_device_type(train_device): + model.fit(X_train_class, y_train_class) + with using_device_type(infer_device): + output = model.predict(X_test_class) + + np.testing.assert_allclose(ref_output, output, rtol=0.15) + + +@pytest.mark.parametrize("train_device", ["cpu", "gpu"]) +@pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) +def test_svr_methods(train_device, infer_device): + ref_model = skSVR() + ref_model.fit(X_train_reg, y_train_reg) + ref_output = ref_model.predict(X_test_reg) + + model = SVR() + with using_device_type(train_device): + model.fit(X_train_reg, y_train_reg) + with using_device_type(infer_device): + output = model.predict(X_test_reg) + + np.testing.assert_allclose(ref_output, output, rtol=0.15) From 8a7e4ba550bc62550ca6f0012f0c89e980ff4d22 Mon Sep 17 00:00:00 2001 From: viclafargue Date: Mon, 7 Oct 2024 19:31:36 +0200 Subject: [PATCH 4/9] SVM single class --- python/cuml/cuml/svm/svc.pyx | 2 ++ python/cuml/cuml/svm/svm_base.pyx | 25 +++++++++++-------- python/cuml/cuml/svm/svr.pyx | 10 +++++--- .../cuml/cuml/tests/test_device_selection.py | 6 +++++ 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/python/cuml/cuml/svm/svc.pyx b/python/cuml/cuml/svm/svc.pyx index 7b5781605d..5207673f24 100644 --- a/python/cuml/cuml/svm/svc.pyx +++ b/python/cuml/cuml/svm/svc.pyx @@ -337,6 +337,8 @@ class SVC(SVMBase, class_weight_ = CumlArrayDescriptor(order='F') + classes__order = 'F' + @device_interop_preparation def __init__(self, *, handle=None, C=1, kernel='rbf', degree=3, gamma='scale', coef0=0.0, tol=1e-3, cache_size=1024.0, diff --git a/python/cuml/cuml/svm/svm_base.pyx b/python/cuml/cuml/svm/svm_base.pyx index d8702bb861..4ab7334fa6 100644 --- a/python/cuml/cuml/svm/svm_base.pyx +++ b/python/cuml/cuml/svm/svm_base.pyx @@ -221,6 +221,10 @@ class SVMBase(UniversalBase, _internal_coef_ = CumlArrayDescriptor(order='F') _unique_labels_ = CumlArrayDescriptor(order='F') + _dual_coef__order = 'F' + support_vectors__order = 'F' + _intercept__order = 'F' + def __init__(self, *, handle=None, C=1, kernel='rbf', degree=3, gamma='auto', coef0=0.0, tol=1e-3, cache_size=1024.0, max_iter=-1, nochange_steps=1000, verbose=False, @@ -409,9 +413,10 @@ class SVMBase(UniversalBase, @_intercept_.setter def _intercept_(self, value): - if isinstance(value, CumlArray): - value = value.to_output('cupy') - self._intercept__ = cupy.ascontiguousarray(value) + if hasattr(self, 'n_classes_') and self.n_classes_ == 2: + value = -1.0 * value.to_output('cupy') + value = input_to_cuml_array(value)[0] + self._intercept__ = value @property def _dual_coef_(self): @@ -419,14 +424,14 @@ class SVMBase(UniversalBase, @_dual_coef_.setter def _dual_coef_(self, value): - if isinstance(value, CumlArray): - value = value.to_output('cupy') - self.dual_coef_ = cupy.ascontiguousarray(value) + if hasattr(self, 'n_classes_') and self.n_classes_ == 2: + value = -1.0 * value.to_output('cupy') + value = input_to_cuml_array(value)[0] + self.dual_coef_ = value @property def support_vectors_(self): - support_vectors = self.support_vectors__.to_output('numpy').astype(np.float64) - return np.ascontiguousarray(support_vectors) + return self.support_vectors__.to_output('numpy').astype(np.float64) @support_vectors_.setter def support_vectors_(self, value): @@ -735,8 +740,8 @@ class SVMBase(UniversalBase, def get_attr_names(self): attr_names = ["_dual_coef_", "fit_status_", "_intercept_", "n_features_in_", "_n_support", "shape_fit_", - "support_", "support_vectors_", - "_probA", "_probB", "_gamma"] + "support_", "support_vectors_", "_probA", + "_probB", "_gamma"] if self.kernel == "linear": attr_names.append("coef_") return attr_names diff --git a/python/cuml/cuml/svm/svr.pyx b/python/cuml/cuml/svm/svr.pyx index b308774653..73f8d9d537 100644 --- a/python/cuml/cuml/svm/svr.pyx +++ b/python/cuml/cuml/svm/svr.pyx @@ -359,6 +359,10 @@ class SVR(SVMBase, RegressorMixin): def get_attr_names(self): return super().get_attr_names() + ["_sparse"] - def gpu_to_cpu(self): - super().gpu_to_cpu() - self._cpu_model._n_support = np.array([self.n_support_], dtype=np.int32) + def cpu_to_gpu(self): + self.dtype = np.float64 + self.n_classes_ = 0 + + super().cpu_to_gpu() + self.n_support_ = self._cpu_model.n_support_ + self._model = self._get_svm_model() diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index 80359d9298..587a9dac79 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -1023,6 +1023,9 @@ def test_dbscan_methods(train_device, infer_device): @pytest.mark.parametrize("train_device", ["cpu", "gpu"]) @pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) def test_svc_methods(train_device, infer_device): + if train_device == "gpu" and infer_device == "cpu": + pytest.skip("GPU training + CPU inference not supported yet") + ref_model = skSVC() ref_model.fit(X_train_class, y_train_class) ref_output = ref_model.predict(X_test_class) @@ -1039,6 +1042,9 @@ def test_svc_methods(train_device, infer_device): @pytest.mark.parametrize("train_device", ["cpu", "gpu"]) @pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) def test_svr_methods(train_device, infer_device): + if train_device == "gpu" and infer_device == "cpu": + pytest.skip("GPU training + CPU inference not supported yet") + ref_model = skSVR() ref_model.fit(X_train_reg, y_train_reg) ref_output = ref_model.predict(X_test_reg) From 01f499678079a72cf114943bf20c625a0cc7c42f Mon Sep 17 00:00:00 2001 From: viclafargue Date: Wed, 9 Oct 2024 17:47:29 +0200 Subject: [PATCH 5/9] completing SVM single class --- python/cuml/cuml/svm/svc.pyx | 6 - python/cuml/cuml/svm/svm_base.pyx | 184 +++++++++--------- .../cuml/cuml/tests/test_device_selection.py | 6 - 3 files changed, 95 insertions(+), 101 deletions(-) diff --git a/python/cuml/cuml/svm/svc.pyx b/python/cuml/cuml/svm/svc.pyx index 5207673f24..abb5f2e8f9 100644 --- a/python/cuml/cuml/svm/svc.pyx +++ b/python/cuml/cuml/svm/svc.pyx @@ -337,8 +337,6 @@ class SVC(SVMBase, class_weight_ = CumlArrayDescriptor(order='F') - classes__order = 'F' - @device_interop_preparation def __init__(self, *, handle=None, C=1, kernel='rbf', degree=3, gamma='scale', coef0=0.0, tol=1e-3, cache_size=1024.0, @@ -526,10 +524,6 @@ class SVC(SVMBase, _array_type, is_sparse = determine_array_type_full(X) self._sparse = is_sparse - # n = int(self.n_classes_ * (self.n_classes_ - 1) / 2) - # self._probA = np.empty(n, dtype=np.float64) - # self._probB = np.empty(n, dtype=np.float64) - if self.probability: return self._fit_proba(X, y, sample_weight) diff --git a/python/cuml/cuml/svm/svm_base.pyx b/python/cuml/cuml/svm/svm_base.pyx index 4ab7334fa6..d732ce1afe 100644 --- a/python/cuml/cuml/svm/svm_base.pyx +++ b/python/cuml/cuml/svm/svm_base.pyx @@ -37,6 +37,8 @@ from cuml.common import using_output_type from cuml.internals.logger import warn from cuml.internals.mixins import FMajorInputTagMixin from cuml.internals.array_sparse import SparseCumlArray, SparseCumlArrayInput +from cuml.internals.mem_type import MemoryType +from cuml.internals.available_devices import is_cuda_available from libcpp cimport bool @@ -216,15 +218,11 @@ class SVMBase(UniversalBase, dual_coef_ = CumlArrayDescriptor(order='F') support_ = CumlArrayDescriptor(order='F') - support_vectors__ = CumlArrayDescriptor(order='F') - _intercept__ = CumlArrayDescriptor(order='F') + support_vectors_ = CumlArrayDescriptor(order='F') + _intercept_ = CumlArrayDescriptor(order='F') _internal_coef_ = CumlArrayDescriptor(order='F') _unique_labels_ = CumlArrayDescriptor(order='F') - _dual_coef__order = 'F' - support_vectors__order = 'F' - _intercept__order = 'F' - def __init__(self, *, handle=None, C=1, kernel='rbf', degree=3, gamma='auto', coef0=0.0, tol=1e-3, cache_size=1024.0, max_iter=-1, nochange_steps=1000, verbose=False, @@ -252,8 +250,8 @@ class SVMBase(UniversalBase, # Attributes (parameters of the fitted model) self.dual_coef_ = None self.support_ = None - self.support_vectors__ = None - self._intercept__ = None + self.support_vectors_ = None + self._intercept_ = None self.n_support_ = None self._c_kernel = self._get_c_kernel(kernel) @@ -347,7 +345,7 @@ class SVMBase(UniversalBase, if self.n_support_ == 0: return cupy.zeros((1, self.n_features_in_), dtype=self.dtype) with using_output_type("cupy"): - return cupy.dot(self.dual_coef_, self.support_vectors__) + return cupy.dot(self.dual_coef_, self.support_vectors_) def _check_is_fitted(self, attr): if not hasattr(self, attr) or (getattr(self, attr) is None): @@ -371,71 +369,16 @@ class SVMBase(UniversalBase, def coef_(self, value): self._internal_coef_ = value - @property - def _probA(self): - if not hasattr(self, '__probA'): - return np.empty(0, dtype=np.float64) - else: - return self.__probA - - @_probA.setter - def _probA(self, value): - self.__probA = value - - @property - def _probB(self): - if not hasattr(self, '__probB'): - return np.empty(0, dtype=np.float64) - else: - return self.__probB - - @_probB.setter - def _probB(self, value): - self.__probB = value - @property @cuml.internals.api_base_return_array_skipall def intercept_(self): - if self._intercept__ is None: + if self._intercept_ is None: raise AttributeError("intercept_ called before fit.") - return self._intercept__ + return self._intercept_ @intercept_.setter def intercept_(self, value): - self._intercept__ = value - - @property - @cuml.internals.api_base_return_array_skipall - def _intercept_(self): - if self._intercept__ is None: - raise AttributeError("intercept_ called before fit.") - return self._intercept__.to_output('numpy').astype(np.float64) - - @_intercept_.setter - def _intercept_(self, value): - if hasattr(self, 'n_classes_') and self.n_classes_ == 2: - value = -1.0 * value.to_output('cupy') - value = input_to_cuml_array(value)[0] - self._intercept__ = value - - @property - def _dual_coef_(self): - return self.dual_coef_.to_output('numpy').astype(np.float64) - - @_dual_coef_.setter - def _dual_coef_(self, value): - if hasattr(self, 'n_classes_') and self.n_classes_ == 2: - value = -1.0 * value.to_output('cupy') - value = input_to_cuml_array(value)[0] - self.dual_coef_ = value - - @property - def support_vectors_(self): - return self.support_vectors__.to_output('numpy').astype(np.float64) - - @support_vectors_.setter - def support_vectors_(self, value): - self.support_vectors__ = value + self._intercept_ = value def _get_kernel_params(self, X=None): """ Wrap the kernel parameters in a KernelParams obtect """ @@ -479,16 +422,16 @@ class SVMBase(UniversalBase, model_f = new SvmModel[float]() model_f.n_support = n_support model_f.n_cols = self.n_features_in_ - model_f.b = self._intercept__.item() + model_f.b = self._intercept_.item() model_f.dual_coefs = \ self.dual_coef_.ptr - if isinstance(self.support_vectors__, SparseCumlArray): - model_f.support_matrix.nnz = self.support_vectors__.nnz - model_f.support_matrix.indptr = self.support_vectors__.indptr.ptr - model_f.support_matrix.indices = self.support_vectors__.indices.ptr - model_f.support_matrix.data = self.support_vectors__.data.ptr + if isinstance(self.support_vectors_, SparseCumlArray): + model_f.support_matrix.nnz = self.support_vectors_.nnz + model_f.support_matrix.indptr = self.support_vectors_.indptr.ptr + model_f.support_matrix.indices = self.support_vectors_.indices.ptr + model_f.support_matrix.data = self.support_vectors_.data.ptr else: - model_f.support_matrix.data = self.support_vectors__.ptr + model_f.support_matrix.data = self.support_vectors_.ptr model_f.support_idx = \ self.support_.ptr model_f.n_classes = self.n_classes_ @@ -502,16 +445,16 @@ class SVMBase(UniversalBase, model_d = new SvmModel[double]() model_d.n_support = n_support model_d.n_cols = self.n_features_in_ - model_d.b = self._intercept__.item() + model_d.b = self._intercept_.item() model_d.dual_coefs = \ self.dual_coef_.ptr - if isinstance(self.support_vectors__, SparseCumlArray): - model_d.support_matrix.nnz = self.support_vectors__.nnz - model_d.support_matrix.indptr = self.support_vectors__.indptr.ptr - model_d.support_matrix.indices = self.support_vectors__.indices.ptr - model_d.support_matrix.data = self.support_vectors__.data.ptr + if isinstance(self.support_vectors_, SparseCumlArray): + model_d.support_matrix.nnz = self.support_vectors_.nnz + model_d.support_matrix.indptr = self.support_vectors_.indptr.ptr + model_d.support_matrix.indices = self.support_vectors_.indices.ptr + model_d.support_matrix.data = self.support_vectors_.data.ptr else: - model_d.support_matrix.data = self.support_vectors__.ptr + model_d.support_matrix.data = self.support_vectors_.ptr model_d.support_idx = \ self.support_.ptr model_d.n_classes = self.n_classes_ @@ -523,7 +466,7 @@ class SVMBase(UniversalBase, return model_d def _unpack_svm_model(self, b, n_support, dual_coefs, support_idx, nnz, indptr, indices, data, n_classes, unique_labels): - self._intercept__ = CumlArray.full(1, b, self.dtype) + self._intercept_ = CumlArray.full(1, b, self.dtype) self.n_support_ = n_support if n_support > 0: @@ -540,7 +483,7 @@ class SVMBase(UniversalBase, order='F') if nnz == -1: - self.support_vectors__ = CumlArray( + self.support_vectors_ = CumlArray( data=data, shape=(self.n_support_, self.n_features_in_), dtype=self.dtype, @@ -565,7 +508,7 @@ class SVMBase(UniversalBase, data=data, nnz=nnz, shape=(self.n_support_, self.n_features_in_)) - self.support_vectors__ = SparseCumlArray(data=sparse_input) + self.support_vectors_ = SparseCumlArray(data=sparse_input) self.n_classes_ = n_classes if self.n_classes_ > 0: @@ -627,7 +570,7 @@ class SVMBase(UniversalBase, # Setting all dims to zero due to issue # https://github.com/rapidsai/cuml/issues/4095 - self.support_vectors__ = CumlArray.empty( + self.support_vectors_ = CumlArray.empty( shape=(0, 0), dtype=self.dtype, order='F') @@ -738,10 +681,8 @@ class SVMBase(UniversalBase, ] def get_attr_names(self): - attr_names = ["_dual_coef_", "fit_status_", "_intercept_", - "n_features_in_", "_n_support", "shape_fit_", - "support_", "support_vectors_", "_probA", - "_probB", "_gamma"] + attr_names = ["fit_status_", "n_features_in_", "shape_fit_", + "support_", "_probA", "_probB", "_gamma"] if self.kernel == "linear": attr_names.append("coef_") return attr_names @@ -758,3 +699,68 @@ class SVMBase(UniversalBase, self.__dict__.update(state) self._model = self._get_svm_model() self._freeSvmBuffers = False + + def gpu_to_cpu(self): + self._cpu_model._n_support = np.array([self.n_support_, 0], dtype=np.int32) + + super().gpu_to_cpu() + + intercept_ = self._intercept_.to_output('numpy').astype(np.float64) + dual_coef_ = self.dual_coef_.to_output('numpy').astype(np.float64) + support_vectors_ = self.support_vectors_.to_output('numpy').astype(np.float64) + + if self.n_classes_ == 2: + intercept_ = -1.0 * intercept_ + dual_coef_ = -1.0 * dual_coef_ + + self._cpu_model._intercept_ = np.array(intercept_, order='C') + self._cpu_model._dual_coef_ = np.array(dual_coef_, order='C') + self._cpu_model.support_vectors_ = np.array(support_vectors_, order='C') + + if hasattr(self, 'n_classes_'): + n = self.n_classes_ * (self.n_classes_ - 1) // 2 + _probA = np.empty(n, dtype=np.float64) + _probB = np.empty(n, dtype=np.float64) + else: + _probA = np.empty(0, dtype=np.float64) + _probB = np.empty(0, dtype=np.float64) + + self._cpu_model._probA = _probA + self._cpu_model._probB = _probB + + def cpu_to_gpu(self): + super().cpu_to_gpu() + + intercept_ = self._cpu_model._intercept_ + dual_coef_ = self._cpu_model._dual_coef_ + + if self.n_classes_ == 2: + intercept_ = -1.0 * intercept_ + dual_coef_ = -1.0 * dual_coef_ + + self._intercept_ = input_to_cuml_array( + intercept_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + order='F')[0] + self.dual_coef_ = input_to_cuml_array( + dual_coef_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + order='F')[0] + self.support_vectors_ = input_to_cuml_array( + self._cpu_model.support_vectors_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + order='F')[0] + + if hasattr(self, 'n_classes_'): + n = self.n_classes_ * (self.n_classes_ - 1) // 2 + _probA = np.empty(n, dtype=np.float64) + _probB = np.empty(n, dtype=np.float64) + else: + _probA = np.empty(0, dtype=np.float64) + _probB = np.empty(0, dtype=np.float64) + + self._probA = _probA + self._probB = _probB diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index 587a9dac79..80359d9298 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -1023,9 +1023,6 @@ def test_dbscan_methods(train_device, infer_device): @pytest.mark.parametrize("train_device", ["cpu", "gpu"]) @pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) def test_svc_methods(train_device, infer_device): - if train_device == "gpu" and infer_device == "cpu": - pytest.skip("GPU training + CPU inference not supported yet") - ref_model = skSVC() ref_model.fit(X_train_class, y_train_class) ref_output = ref_model.predict(X_test_class) @@ -1042,9 +1039,6 @@ def test_svc_methods(train_device, infer_device): @pytest.mark.parametrize("train_device", ["cpu", "gpu"]) @pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) def test_svr_methods(train_device, infer_device): - if train_device == "gpu" and infer_device == "cpu": - pytest.skip("GPU training + CPU inference not supported yet") - ref_model = skSVR() ref_model.fit(X_train_reg, y_train_reg) ref_output = ref_model.predict(X_test_reg) From 9e49d9eebad213107f2298bfd932bb93ee001606 Mon Sep 17 00:00:00 2001 From: viclafargue Date: Fri, 22 Nov 2024 19:32:14 +0100 Subject: [PATCH 6/9] Support for multiclass --- python/cuml/cuml/multiclass/multiclass.py | 50 +----- python/cuml/cuml/svm/__init__.py | 4 +- python/cuml/cuml/svm/svc.pyx | 150 ++++++++++++++++-- python/cuml/cuml/svm/svm_base.pyx | 87 +++++----- python/cuml/cuml/svm/svr.pyx | 3 - .../cuml/cuml/tests/test_device_selection.py | 41 +++-- 6 files changed, 223 insertions(+), 112 deletions(-) diff --git a/python/cuml/cuml/multiclass/multiclass.py b/python/cuml/cuml/multiclass/multiclass.py index 7ae123b2ff..225f202fb9 100644 --- a/python/cuml/cuml/multiclass/multiclass.py +++ b/python/cuml/cuml/multiclass/multiclass.py @@ -113,27 +113,17 @@ def __init__( self.strategy = strategy self.estimator = estimator - @property - @cuml.internals.api_base_return_array_skipall - def classes_(self): - return self.multiclass_estimator.classes_ - - @classes_.setter - def classes_(self, value): + if not has_sklearn(): + raise ImportError( + "Scikit-learn is needed to use " + "MulticlassClassifier derived classes." + ) import sklearn.multiclass if self.strategy == "ovr": self.multiclass_estimator = sklearn.multiclass.OneVsRestClassifier( self.estimator, n_jobs=None ) - from sklearn.preprocessing import LabelBinarizer - - self.multiclass_estimator.label_binarizer_ = LabelBinarizer( - sparse_output=True - ) - self.multiclass_estimator.label_binarizer_.fit( - value.to_output("numpy") - ) elif self.strategy == "ovo": self.multiclass_estimator = sklearn.multiclass.OneVsOneClassifier( self.estimator, n_jobs=None @@ -145,41 +135,17 @@ def classes_(self, value): + ", must be one of " '{"ovr", "ovo"}' ) - self.multiclass_estimator.classes_ = value @property - @cuml.internals.api_base_return_any_skipall - def n_classes_(self): - return self.multiclass_estimator.n_classes_ + @cuml.internals.api_base_return_array_skipall + def classes_(self): + return self.multiclass_estimator.classes_ @generate_docstring(y="dense_anydtype") def fit(self, X, y) -> "MulticlassClassifier": """ Fit a multiclass classifier. """ - if not has_sklearn(): - raise ImportError( - "Scikit-learn is needed to use " - "MulticlassClassifier derived classes." - ) - import sklearn.multiclass - - if self.strategy == "ovr": - self.multiclass_estimator = sklearn.multiclass.OneVsRestClassifier( - self.estimator, n_jobs=None - ) - elif self.strategy == "ovo": - self.multiclass_estimator = sklearn.multiclass.OneVsOneClassifier( - self.estimator, n_jobs=None - ) - else: - raise ValueError( - "Invalid multiclass strategy " - + str(self.strategy) - + ", must be one of " - '{"ovr", "ovo"}' - ) - X = input_to_host_array_with_sparse_support(X) y = input_to_host_array(y).array diff --git a/python/cuml/cuml/svm/__init__.py b/python/cuml/cuml/svm/__init__.py index b5aa2705f7..f71dc43c6a 100644 --- a/python/cuml/cuml/svm/__init__.py +++ b/python/cuml/cuml/svm/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019-2021, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from cuml.svm.svc import SVC +from cuml.svm.svc import SVC, cpuModelSVC from cuml.svm.svr import SVR from cuml.svm.linear_svc import LinearSVC from cuml.svm.linear_svr import LinearSVR diff --git a/python/cuml/cuml/svm/svc.pyx b/python/cuml/cuml/svm/svc.pyx index abb5f2e8f9..82684d51c1 100644 --- a/python/cuml/cuml/svm/svc.pyx +++ b/python/cuml/cuml/svm/svc.pyx @@ -46,6 +46,35 @@ from cuml.internals.import_utils import has_sklearn from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.api_decorators import device_interop_preparation, enable_device_interop +from sklearn.svm import SVC as skSVC +from sklearn.preprocessing import LabelBinarizer +from sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier +from cuml.internals.mem_type import MemoryType +from cuml.internals.available_devices import is_cuda_available + + +class cpuModelSVC(skSVC): + def fit(self, X, y, sample_weight=None): + self.classes_ = np.unique(y) + self.n_classes_ = len(self.classes_) + + if self.n_classes_ == 2: + super().fit(X, y, sample_weight) + else: + params = self.get_params() + if self.decision_function_shape == 'ovr': + self.multi_class_model = OneVsRestClassifier(skSVC(**params)) + elif self.decision_function_shape == 'ovo': + self.multi_class_model = OneVsOneClassifier(skSVC(**params)) + self.multi_class_model.fit(X, y) + + def predict(self, X): + if self.n_classes_ == 2: + return super().predict(X) + else: + return self.multi_class_model.predict(X) + + if has_sklearn(): from cuml.multiclass import MulticlassClassifier from sklearn.calibration import CalibratedClassifierCV @@ -333,7 +362,7 @@ class SVC(SVMBase, """ - _cpu_estimator_import_path = 'sklearn.svm.SVC' + _cpu_estimator_import_path = 'cuml.svm.cpuModelSVC' class_weight_ = CumlArrayDescriptor(order='F') @@ -750,14 +779,14 @@ class SVC(SVMBase, def cpu_to_gpu(self): self.dtype = np.float64 + self.target_dtype = np.int64 self.probability = self._cpu_model.probability - self.n_classes_ = len(self._cpu_model.classes_) + self.n_classes_ = self._cpu_model.n_classes_ if self.probability: - params = self.get_params() - params["probability"] = False - params["output_type"] = "numpy" - self.prob_svc = CalibratedClassifierCV(SVC(**params), cv=5, method='sigmoid') + pass + elif self.n_classes_ == 2: + super().cpu_to_gpu() elif self.n_classes_ > 2: if not hasattr(self, 'multiclass_svc'): params = self.get_params() @@ -766,7 +795,108 @@ class SVC(SVMBase, MulticlassClassifier(estimator=SVC(**params), handle=self.handle, verbose=self.verbose, output_type=self.output_type, strategy=strategy) - else: - super().cpu_to_gpu() - self.n_support_ = self._cpu_model.n_support_ - self._model = self._get_svm_model() + + def turn_cpu_into_gpu(cpu_est): + gpu_est = SVC(**params) + gpu_est.dtype = np.float64 + gpu_est.target_dtype = np.int64 + gpu_est.n_classes_ = 2 + + # n_classes == 2 in this case + intercept_ = -1.0 * cpu_est._intercept_ + dual_coef_ = -1.0 * cpu_est._dual_coef_ + + gpu_est.n_support_ = cpu_est.n_support_.sum() + + gpu_est._intercept_ = input_to_cuml_array( + intercept_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + gpu_est.dual_coef_ = input_to_cuml_array( + dual_coef_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + gpu_est.support_ = input_to_cuml_array( + cpu_est.support_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.int32, + order='F')[0] + gpu_est.support_vectors_ = input_to_cuml_array( + cpu_est.support_vectors_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + gpu_est._unique_labels_ = input_to_cuml_array( + np.array(cpu_est.classes_, dtype=np.float64), + deepcopy=True, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + + gpu_est._probA = cp.empty(0, dtype=np.float64) + gpu_est._probB = cp.empty(0, dtype=np.float64) + gpu_est._gamma = cpu_est._gamma + gpu_est.fit_status_ = cpu_est.fit_status_ + gpu_est.n_features_in_ = cpu_est.n_features_in_ + gpu_est._sparse = cpu_est._sparse + + gpu_est._model = gpu_est._get_svm_model() + return gpu_est + + self.multiclass_svc.multiclass_estimator.classes_ = self._cpu_model.classes_ + estimators = self._cpu_model.multi_class_model.estimators_ + self.multiclass_svc.multiclass_estimator.estimators_ = [turn_cpu_into_gpu(est) for est in estimators] + + if strategy == 'ovr': + self.multiclass_svc.multiclass_estimator.label_binarizer_ = LabelBinarizer(sparse_output=True) + self.multiclass_svc.multiclass_estimator.label_binarizer_.fit(self._cpu_model.classes_) + elif strategy == 'ovo': + self.multiclass_svc.multiclass_estimator.pairwise_indices_ = None + + def gpu_to_cpu(self): + self._cpu_model.n_classes_ = self.n_classes_ + + if self.probability: + pass + elif self.n_classes_ == 2: + super().gpu_to_cpu() + elif self.n_classes_ > 2: + estimators = self.multiclass_svc.multiclass_estimator.estimators_ + classes = self.classes_.to_output('numpy').astype(np.int32) + + if self.decision_function_shape == 'ovr': + self._cpu_model.multi_class_model = OneVsRestClassifier(skSVC) + self._cpu_model.multi_class_model.label_binarizer_ = LabelBinarizer(sparse_output=True) + self._cpu_model.multi_class_model.label_binarizer_.fit(classes) + elif self.decision_function_shape == 'ovo': + self._cpu_model.multi_class_model = OneVsOneClassifier(skSVC) + self._cpu_model.multi_class_model.pairwise_indices_ = None + + params = self.get_params() + params = {key: value for key, value, in params.items() if key in self._cpu_hyperparams} + + def turn_gpu_into_cpu(gpu_est): + cpu_est = skSVC(**params) + cpu_est.support_ = gpu_est.support_.to_output('numpy').astype(np.int32) + cpu_est.support_vectors_ = np.ascontiguousarray(gpu_est.support_vectors_.to_output('numpy').astype(np.float64)) + cpu_est._n_support = np.array([gpu_est.n_support_, 0]).astype(np.int32) + cpu_est._dual_coef_ = -1.0 * np.ascontiguousarray(gpu_est.dual_coef_.to_output('numpy').astype(np.float64)) + cpu_est._intercept_ = -1.0 * gpu_est.intercept_.to_output('numpy').astype(np.float64) + cpu_est.classes_ = gpu_est.classes_.to_output('numpy').astype(np.int32) + cpu_est.n_classes_ = 2 + cpu_est._probA = np.empty(0, dtype=np.float64) + cpu_est._probB = np.empty(0, dtype=np.float64) + cpu_est.fit_status_ = gpu_est.fit_status_ + cpu_est._sparse = gpu_est._sparse + cpu_est._gamma = gpu_est._gamma + return cpu_est + + self._cpu_model.multi_class_model.estimators_ = [turn_gpu_into_cpu(est) for est in estimators] + self._cpu_model.multi_class_model.classes_ = classes diff --git a/python/cuml/cuml/svm/svm_base.pyx b/python/cuml/cuml/svm/svm_base.pyx index d732ce1afe..86854ac718 100644 --- a/python/cuml/cuml/svm/svm_base.pyx +++ b/python/cuml/cuml/svm/svm_base.pyx @@ -434,12 +434,13 @@ class SVMBase(UniversalBase, model_f.support_matrix.data = self.support_vectors_.ptr model_f.support_idx = \ self.support_.ptr - model_f.n_classes = self.n_classes_ - if self.n_classes_ > 0: - model_f.unique_labels = \ - self._unique_labels_.ptr - else: - model_f.unique_labels = NULL + if hasattr(self, 'n_classes_'): + model_f.n_classes = self.n_classes_ + if self.n_classes_ > 0: + model_f.unique_labels = \ + self._unique_labels_.ptr + else: + model_f.unique_labels = NULL return model_f else: model_d = new SvmModel[double]() @@ -457,12 +458,13 @@ class SVMBase(UniversalBase, model_d.support_matrix.data = self.support_vectors_.ptr model_d.support_idx = \ self.support_.ptr - model_d.n_classes = self.n_classes_ - if self.n_classes_ > 0: - model_d.unique_labels = \ - self._unique_labels_.ptr - else: - model_d.unique_labels = NULL + if hasattr(self, 'n_classes_'): + model_d.n_classes = self.n_classes_ + if self.n_classes_ > 0: + model_d.unique_labels = \ + self._unique_labels_.ptr + else: + model_d.unique_labels = NULL return model_d def _unpack_svm_model(self, b, n_support, dual_coefs, support_idx, nnz, indptr, indices, data, n_classes, unique_labels): @@ -681,8 +683,7 @@ class SVMBase(UniversalBase, ] def get_attr_names(self): - attr_names = ["fit_status_", "n_features_in_", "shape_fit_", - "support_", "_probA", "_probB", "_gamma"] + attr_names = ["fit_status_", "n_features_in_", "shape_fit_"] if self.kernel == "linear": attr_names.append("coef_") return attr_names @@ -701,40 +702,36 @@ class SVMBase(UniversalBase, self._freeSvmBuffers = False def gpu_to_cpu(self): - self._cpu_model._n_support = np.array([self.n_support_, 0], dtype=np.int32) - super().gpu_to_cpu() - intercept_ = self._intercept_.to_output('numpy').astype(np.float64) - dual_coef_ = self.dual_coef_.to_output('numpy').astype(np.float64) - support_vectors_ = self.support_vectors_.to_output('numpy').astype(np.float64) + intercept_ = self._intercept_.to_output('numpy') + dual_coef_ = self.dual_coef_.to_output('numpy') + support_= self.support_.to_output('numpy') + support_vectors_ = self.support_vectors_.to_output('numpy') - if self.n_classes_ == 2: + if hasattr(self, 'n_classes_') and self.n_classes_ == 2: intercept_ = -1.0 * intercept_ dual_coef_ = -1.0 * dual_coef_ - self._cpu_model._intercept_ = np.array(intercept_, order='C') - self._cpu_model._dual_coef_ = np.array(dual_coef_, order='C') - self._cpu_model.support_vectors_ = np.array(support_vectors_, order='C') - - if hasattr(self, 'n_classes_'): - n = self.n_classes_ * (self.n_classes_ - 1) // 2 - _probA = np.empty(n, dtype=np.float64) - _probB = np.empty(n, dtype=np.float64) - else: - _probA = np.empty(0, dtype=np.float64) - _probB = np.empty(0, dtype=np.float64) + self._cpu_model._n_support = np.array([self.n_support_, 0], dtype=np.int32) + self._cpu_model._intercept_ = np.ascontiguousarray(intercept_, dtype=np.float64) + self._cpu_model._dual_coef_ = np.ascontiguousarray(dual_coef_, dtype=np.float64) + self._cpu_model.support_ = np.ascontiguousarray(support_, dtype=np.int32) + self._cpu_model.support_vectors_ = np.ascontiguousarray(support_vectors_, dtype=np.float64) - self._cpu_model._probA = _probA - self._cpu_model._probB = _probB + self._cpu_model._probA = np.empty(0, dtype=np.float64) + self._cpu_model._probB = np.empty(0, dtype=np.float64) + self._cpu_model._gamma = self._gamma def cpu_to_gpu(self): super().cpu_to_gpu() + self.n_support_ = self._cpu_model.n_support_ + intercept_ = self._cpu_model._intercept_ dual_coef_ = self._cpu_model._dual_coef_ - if self.n_classes_ == 2: + if hasattr(self, 'n_classes_') and self.n_classes_ == 2: intercept_ = -1.0 * intercept_ dual_coef_ = -1.0 * dual_coef_ @@ -742,25 +739,29 @@ class SVMBase(UniversalBase, intercept_, convert_to_mem_type=(MemoryType.host, MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, order='F')[0] self.dual_coef_ = input_to_cuml_array( dual_coef_, convert_to_mem_type=(MemoryType.host, MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + self.support_ = input_to_cuml_array( + self._cpu_model.support_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.int32, order='F')[0] self.support_vectors_ = input_to_cuml_array( self._cpu_model.support_vectors_, convert_to_mem_type=(MemoryType.host, MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, order='F')[0] - if hasattr(self, 'n_classes_'): - n = self.n_classes_ * (self.n_classes_ - 1) // 2 - _probA = np.empty(n, dtype=np.float64) - _probB = np.empty(n, dtype=np.float64) - else: - _probA = np.empty(0, dtype=np.float64) - _probB = np.empty(0, dtype=np.float64) + self._probA = np.empty(0, dtype=np.float64) + self._probB = np.empty(0, dtype=np.float64) + self._gamma = self._cpu_model._gamma - self._probA = _probA - self._probB = _probB + self._model = self._get_svm_model() diff --git a/python/cuml/cuml/svm/svr.pyx b/python/cuml/cuml/svm/svr.pyx index 73f8d9d537..d458f05cab 100644 --- a/python/cuml/cuml/svm/svr.pyx +++ b/python/cuml/cuml/svm/svr.pyx @@ -361,8 +361,5 @@ class SVR(SVMBase, RegressorMixin): def cpu_to_gpu(self): self.dtype = np.float64 - self.n_classes_ = 0 super().cpu_to_gpu() - self.n_support_ = self._cpu_model.n_support_ - self._model = self._get_svm_model() diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index b4db3aaeb6..53dee0bf44 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -146,12 +146,12 @@ def make_reg_dataset(): ) -def make_class_dataset(): +def make_class_dataset(n_classes): X, y = make_classification( n_samples=2000, n_features=20, n_informative=18, - n_classes=2, + n_classes=n_classes, random_state=0, ) X_train, X_test = X[:1800], X[1800:] @@ -181,7 +181,10 @@ def make_blob_dataset(): X_train_reg, y_train_reg, X_test_reg = make_reg_dataset() -X_train_class, y_train_class, X_test_class = make_class_dataset() +X_train_class, y_train_class, X_test_class = make_class_dataset(2) +X_train_multiclass, y_train_multiclass, X_test_multiclass = make_class_dataset( + 5 +) X_train_blob, y_train_blob, X_test_blob = make_blob_dataset() @@ -1036,18 +1039,32 @@ def test_dbscan_methods(train_device, infer_device): @pytest.mark.parametrize("train_device", ["cpu", "gpu"]) @pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) -def test_svc_methods(train_device, infer_device): - ref_model = skSVC() - ref_model.fit(X_train_class, y_train_class) - ref_output = ref_model.predict(X_test_class) - - model = SVC() +@pytest.mark.parametrize("decision_function_shape", ["ovo", "ovr"]) +@pytest.mark.parametrize("class_type", ["single_class", "multi_class"]) +def test_svc_methods( + train_device, infer_device, decision_function_shape, class_type +): + if class_type == "single_class": + X_train = X_train_class + y_train = y_train_class + X_test = X_test_class + elif class_type == "multi_class": + X_train = X_train_multiclass + y_train = y_train_multiclass + X_test = X_test_multiclass + + ref_model = skSVC(decision_function_shape=decision_function_shape) + ref_model.fit(X_train, y_train) + ref_output = ref_model.predict(X_test) + + model = SVC(decision_function_shape=decision_function_shape) with using_device_type(train_device): - model.fit(X_train_class, y_train_class) + model.fit(X_train, y_train) with using_device_type(infer_device): - output = model.predict(X_test_class) + output = model.predict(X_test) - np.testing.assert_allclose(ref_output, output, rtol=0.15) + correct_percentage = (ref_output == output).sum() / ref_output.size + assert correct_percentage > 0.9 @pytest.mark.parametrize("train_device", ["cpu", "gpu"]) From fbd0875b1847b5b17fae4ad6a3dd7ffdfc55405a Mon Sep 17 00:00:00 2001 From: viclafargue Date: Mon, 2 Dec 2024 18:58:30 +0100 Subject: [PATCH 7/9] Enabling probability generation case --- python/cuml/cuml/svm/svc.pyx | 247 ++++++++++++------ .../cuml/cuml/tests/test_device_selection.py | 17 +- 2 files changed, 182 insertions(+), 82 deletions(-) diff --git a/python/cuml/cuml/svm/svc.pyx b/python/cuml/cuml/svm/svc.pyx index 354fb55a76..67080635ca 100644 --- a/python/cuml/cuml/svm/svc.pyx +++ b/python/cuml/cuml/svm/svc.pyx @@ -58,7 +58,23 @@ class cpuModelSVC(skSVC): self.classes_ = np.unique(y) self.n_classes_ = len(self.classes_) - if self.n_classes_ == 2: + if self.probability: + params = self.get_params() + params["probability"] = False + + if self.n_classes_ == 2: + estimator = skSVC(**params) + else: + if self.decision_function_shape == 'ovr': + estimator = OneVsRestClassifier(skSVC(**params)) + elif self.decision_function_shape == 'ovo': + estimator = OneVsOneClassifier(skSVC(**params)) + + self.prob_svc = CalibratedClassifierCV(estimator, + cv=5, + method='sigmoid') + self.prob_svc.fit(X, y) + elif self.n_classes_ == 2: super().fit(X, y, sample_weight) else: params = self.get_params() @@ -69,7 +85,9 @@ class cpuModelSVC(skSVC): self.multi_class_model.fit(X, y) def predict(self, X): - if self.n_classes_ == 2: + if self.probability: + return self.prob_svc.predict(X) + elif self.n_classes_ == 2: return super().predict(X) else: return self.multi_class_model.predict(X) @@ -512,7 +530,15 @@ class SVC(SVMBase, raise RuntimeError( "Scikit-learn is needed to use SVM probabilities") - self.prob_svc = CalibratedClassifierCV(SVC(**params), + if self.n_classes_ == 2: + estimator = SVC(**params) + else: + if self.decision_function_shape == 'ovr': + estimator = OneVsRestClassifier(SVC(**params)) + elif self.decision_function_shape == 'ovo': + estimator = OneVsOneClassifier(SVC(**params)) + + self.prob_svc = CalibratedClassifierCV(estimator, cv=5, method='sigmoid') @@ -785,8 +811,95 @@ class SVC(SVMBase, self.probability = self._cpu_model.probability self.n_classes_ = self._cpu_model.n_classes_ + def turn_cpu_into_gpu(cpu_est, params): + gpu_est = SVC(**params) + gpu_est.dtype = np.float64 + gpu_est.target_dtype = np.int64 + gpu_est.n_classes_ = 2 + + # n_classes == 2 in this case + intercept_ = -1.0 * cpu_est._intercept_ + dual_coef_ = -1.0 * cpu_est._dual_coef_ + + gpu_est.n_support_ = cpu_est.n_support_.sum() + + gpu_est._intercept_ = input_to_cuml_array( + intercept_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + gpu_est.dual_coef_ = input_to_cuml_array( + dual_coef_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + gpu_est.support_ = input_to_cuml_array( + cpu_est.support_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.int32, + order='F')[0] + gpu_est.support_vectors_ = input_to_cuml_array( + cpu_est.support_vectors_, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + gpu_est._unique_labels_ = input_to_cuml_array( + np.array(cpu_est.classes_, dtype=np.float64), + deepcopy=True, + convert_to_mem_type=(MemoryType.host, + MemoryType.device)[is_cuda_available()], + convert_to_dtype=np.float64, + order='F')[0] + + gpu_est._probA = cp.empty(0, dtype=np.float64) + gpu_est._probB = cp.empty(0, dtype=np.float64) + gpu_est._gamma = cpu_est._gamma + gpu_est.fit_status_ = cpu_est.fit_status_ + gpu_est.n_features_in_ = cpu_est.n_features_in_ + gpu_est._sparse = cpu_est._sparse + + gpu_est._model = gpu_est._get_svm_model() + return gpu_est + if self.probability: - pass + if not hasattr(self, 'prob_svc'): + classes = self._cpu_model.classes_ + + def convert_calibrator(cpu_calibrator, params): + import copy + gpu_calibrator = copy.copy(cpu_calibrator) + cpu_est = cpu_calibrator.estimator + if isinstance(cpu_est, skSVC): + gpu_est = turn_cpu_into_gpu(cpu_est, params) + else: + if self.decision_function_shape == 'ovr': + gpu_est = OneVsRestClassifier(SVC) + gpu_est.label_binarizer_ = LabelBinarizer(sparse_output=True) + gpu_est.label_binarizer_.fit(classes) + elif self.decision_function_shape == 'ovo': + gpu_est = OneVsOneClassifier(SVC) + gpu_est.pairwise_indices_ = None + gpu_est.classes_ = classes + estimators = cpu_est.estimators_ + gpu_est.estimators_ = [turn_cpu_into_gpu(est, params) for est in estimators] + + gpu_calibrator.estimator = gpu_est + return gpu_calibrator + + params = self.get_params() + params = {key: value for key, value, in params.items() if key in self._cpu_hyperparams} + params["probability"] = False + params["output_type"] = "numpy" + self.prob_svc = CalibratedClassifierCV(SVC(**params), + cv=5, + method='sigmoid') + self.prob_svc.classes_ = classes + calibrators = self._cpu_model.prob_svc.calibrated_classifiers_ + self.prob_svc.calibrated_classifiers_ = [convert_calibrator(cal, params) for cal in calibrators] elif self.n_classes_ == 2: super().cpu_to_gpu() elif self.n_classes_ > 2: @@ -798,63 +911,9 @@ class SVC(SVMBase, verbose=self.verbose, output_type=self.output_type, strategy=strategy) - def turn_cpu_into_gpu(cpu_est): - gpu_est = SVC(**params) - gpu_est.dtype = np.float64 - gpu_est.target_dtype = np.int64 - gpu_est.n_classes_ = 2 - - # n_classes == 2 in this case - intercept_ = -1.0 * cpu_est._intercept_ - dual_coef_ = -1.0 * cpu_est._dual_coef_ - - gpu_est.n_support_ = cpu_est.n_support_.sum() - - gpu_est._intercept_ = input_to_cuml_array( - intercept_, - convert_to_mem_type=(MemoryType.host, - MemoryType.device)[is_cuda_available()], - convert_to_dtype=np.float64, - order='F')[0] - gpu_est.dual_coef_ = input_to_cuml_array( - dual_coef_, - convert_to_mem_type=(MemoryType.host, - MemoryType.device)[is_cuda_available()], - convert_to_dtype=np.float64, - order='F')[0] - gpu_est.support_ = input_to_cuml_array( - cpu_est.support_, - convert_to_mem_type=(MemoryType.host, - MemoryType.device)[is_cuda_available()], - convert_to_dtype=np.int32, - order='F')[0] - gpu_est.support_vectors_ = input_to_cuml_array( - cpu_est.support_vectors_, - convert_to_mem_type=(MemoryType.host, - MemoryType.device)[is_cuda_available()], - convert_to_dtype=np.float64, - order='F')[0] - gpu_est._unique_labels_ = input_to_cuml_array( - np.array(cpu_est.classes_, dtype=np.float64), - deepcopy=True, - convert_to_mem_type=(MemoryType.host, - MemoryType.device)[is_cuda_available()], - convert_to_dtype=np.float64, - order='F')[0] - - gpu_est._probA = cp.empty(0, dtype=np.float64) - gpu_est._probB = cp.empty(0, dtype=np.float64) - gpu_est._gamma = cpu_est._gamma - gpu_est.fit_status_ = cpu_est.fit_status_ - gpu_est.n_features_in_ = cpu_est.n_features_in_ - gpu_est._sparse = cpu_est._sparse - - gpu_est._model = gpu_est._get_svm_model() - return gpu_est - self.multiclass_svc.multiclass_estimator.classes_ = self._cpu_model.classes_ estimators = self._cpu_model.multi_class_model.estimators_ - self.multiclass_svc.multiclass_estimator.estimators_ = [turn_cpu_into_gpu(est) for est in estimators] + self.multiclass_svc.multiclass_estimator.estimators_ = [turn_cpu_into_gpu(est, params) for est in estimators] if strategy == 'ovr': self.multiclass_svc.multiclass_estimator.label_binarizer_ = LabelBinarizer(sparse_output=True) @@ -865,8 +924,55 @@ class SVC(SVMBase, def gpu_to_cpu(self): self._cpu_model.n_classes_ = self.n_classes_ + def turn_gpu_into_cpu(gpu_est, params): + cpu_est = skSVC(**params) + cpu_est.support_ = gpu_est.support_.to_output('numpy').astype(np.int32) + cpu_est.support_vectors_ = np.ascontiguousarray(gpu_est.support_vectors_.to_output('numpy').astype(np.float64)) + cpu_est._n_support = np.array([gpu_est.n_support_, 0]).astype(np.int32) + cpu_est._dual_coef_ = -1.0 * np.ascontiguousarray(gpu_est.dual_coef_.to_output('numpy').astype(np.float64)) + cpu_est._intercept_ = -1.0 * gpu_est.intercept_.to_output('numpy').astype(np.float64) + cpu_est.classes_ = gpu_est.classes_.to_output('numpy').astype(np.int32) + cpu_est.n_classes_ = 2 + cpu_est._probA = np.empty(0, dtype=np.float64) + cpu_est._probB = np.empty(0, dtype=np.float64) + cpu_est.fit_status_ = gpu_est.fit_status_ + cpu_est._sparse = gpu_est._sparse + cpu_est._gamma = gpu_est._gamma + return cpu_est + if self.probability: - pass + if not hasattr(self._cpu_model, 'prob_svc'): + def convert_calibrator(gpu_calibrator, params): + import copy + cpu_calibrator = copy.copy(gpu_calibrator) + gpu_est = gpu_calibrator.estimator + if isinstance(gpu_est, SVC): + cpu_est = turn_gpu_into_cpu(gpu_est, params) + else: + classes = self.classes_.to_output('numpy').astype(np.int32) + if self.decision_function_shape == 'ovr': + cpu_est = OneVsRestClassifier(skSVC) + cpu_est.label_binarizer_ = LabelBinarizer(sparse_output=True) + cpu_est.label_binarizer_.fit(classes) + elif self.decision_function_shape == 'ovo': + cpu_est = OneVsOneClassifier(skSVC) + cpu_est.pairwise_indices_ = None + cpu_est.classes_ = classes + estimators = gpu_est.estimators_ + cpu_est.estimators_ = [turn_gpu_into_cpu(est, params) for est in estimators] + + cpu_calibrator.estimator = cpu_est + return cpu_calibrator + + params = self.get_params() + params = {key: value for key, value, in params.items() if key in self._cpu_hyperparams} + params["probability"] = False + self._cpu_model.prob_svc = CalibratedClassifierCV(skSVC(**params), + cv=5, + method='sigmoid') + self._cpu_model.prob_svc.classes_ = self.classes_ + calibrators = self.prob_svc.calibrated_classifiers_ + self._cpu_model.prob_svc.calibrated_classifiers_ = [convert_calibrator(cal, params) for cal in calibrators] elif self.n_classes_ == 2: super().gpu_to_cpu() elif self.n_classes_ > 2: @@ -880,25 +986,8 @@ class SVC(SVMBase, elif self.decision_function_shape == 'ovo': self._cpu_model.multi_class_model = OneVsOneClassifier(skSVC) self._cpu_model.multi_class_model.pairwise_indices_ = None + self._cpu_model.multi_class_model.classes_ = classes params = self.get_params() params = {key: value for key, value, in params.items() if key in self._cpu_hyperparams} - - def turn_gpu_into_cpu(gpu_est): - cpu_est = skSVC(**params) - cpu_est.support_ = gpu_est.support_.to_output('numpy').astype(np.int32) - cpu_est.support_vectors_ = np.ascontiguousarray(gpu_est.support_vectors_.to_output('numpy').astype(np.float64)) - cpu_est._n_support = np.array([gpu_est.n_support_, 0]).astype(np.int32) - cpu_est._dual_coef_ = -1.0 * np.ascontiguousarray(gpu_est.dual_coef_.to_output('numpy').astype(np.float64)) - cpu_est._intercept_ = -1.0 * gpu_est.intercept_.to_output('numpy').astype(np.float64) - cpu_est.classes_ = gpu_est.classes_.to_output('numpy').astype(np.int32) - cpu_est.n_classes_ = 2 - cpu_est._probA = np.empty(0, dtype=np.float64) - cpu_est._probB = np.empty(0, dtype=np.float64) - cpu_est.fit_status_ = gpu_est.fit_status_ - cpu_est._sparse = gpu_est._sparse - cpu_est._gamma = gpu_est._gamma - return cpu_est - - self._cpu_model.multi_class_model.estimators_ = [turn_gpu_into_cpu(est) for est in estimators] - self._cpu_model.multi_class_model.classes_ = classes + self._cpu_model.multi_class_model.estimators_ = [turn_gpu_into_cpu(est, params) for est in estimators] diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index 53dee0bf44..63aeb0a694 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -1041,8 +1041,13 @@ def test_dbscan_methods(train_device, infer_device): @pytest.mark.parametrize("infer_device", ["cpu", "gpu"]) @pytest.mark.parametrize("decision_function_shape", ["ovo", "ovr"]) @pytest.mark.parametrize("class_type", ["single_class", "multi_class"]) +@pytest.mark.parametrize("probability", [True, False]) def test_svc_methods( - train_device, infer_device, decision_function_shape, class_type + train_device, + infer_device, + decision_function_shape, + class_type, + probability, ): if class_type == "single_class": X_train = X_train_class @@ -1053,11 +1058,17 @@ def test_svc_methods( y_train = y_train_multiclass X_test = X_test_multiclass - ref_model = skSVC(decision_function_shape=decision_function_shape) + ref_model = skSVC( + probability=probability, + decision_function_shape=decision_function_shape, + ) ref_model.fit(X_train, y_train) ref_output = ref_model.predict(X_test) - model = SVC(decision_function_shape=decision_function_shape) + model = SVC( + probability=probability, + decision_function_shape=decision_function_shape, + ) with using_device_type(train_device): model.fit(X_train, y_train) with using_device_type(infer_device): From de202a4f28bf3af97789afab97a4fb263481cc5b Mon Sep 17 00:00:00 2001 From: viclafargue Date: Fri, 6 Dec 2024 15:44:06 +0100 Subject: [PATCH 8/9] fix --- python/cuml/cuml/svm/svc.pyx | 28 +++++++++++++++++-- .../cuml/cuml/tests/test_device_selection.py | 24 ++++++++++++---- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/python/cuml/cuml/svm/svc.pyx b/python/cuml/cuml/svm/svc.pyx index 67080635ca..1cdc8638ee 100644 --- a/python/cuml/cuml/svm/svc.pyx +++ b/python/cuml/cuml/svm/svc.pyx @@ -69,6 +69,8 @@ class cpuModelSVC(skSVC): estimator = OneVsRestClassifier(skSVC(**params)) elif self.decision_function_shape == 'ovo': estimator = OneVsOneClassifier(skSVC(**params)) + else: + raise ValueError self.prob_svc = CalibratedClassifierCV(estimator, cv=5, @@ -82,6 +84,8 @@ class cpuModelSVC(skSVC): self.multi_class_model = OneVsRestClassifier(skSVC(**params)) elif self.decision_function_shape == 'ovo': self.multi_class_model = OneVsOneClassifier(skSVC(**params)) + else: + raise ValueError self.multi_class_model.fit(X, y) def predict(self, X): @@ -92,6 +96,14 @@ class cpuModelSVC(skSVC): else: return self.multi_class_model.predict(X) + def predict_proba(self, X): + if self.probability: + return self.prob_svc.predict_proba(X) + elif self.n_classes_ == 2: + return super().predict_proba(X) + else: + return self.multi_class_model.predict_proba(X) + if has_sklearn(): from cuml.multiclass import MulticlassClassifier @@ -485,7 +497,6 @@ class SVC(SVMBase, params = self.get_params() strategy = params.pop('decision_function_shape', 'ovo') - self.multiclass_svc = MulticlassClassifier( estimator=SVC(**params), handle=self.handle, verbose=self.verbose, output_type=self.output_type, strategy=strategy) @@ -537,6 +548,8 @@ class SVC(SVMBase, estimator = OneVsRestClassifier(SVC(**params)) elif self.decision_function_shape == 'ovo': estimator = OneVsOneClassifier(SVC(**params)) + else: + raise ValueError self.prob_svc = CalibratedClassifierCV(estimator, cv=5, @@ -793,8 +806,7 @@ class SVC(SVMBase, @classmethod def _get_param_names(cls): params = super()._get_param_names() + \ - ["probability", "random_state", "class_weight", - "decision_function_shape"] + ["probability", "random_state", "class_weight", "decision_function_shape"] # Ignore "epsilon" since its not used in the constructor if ("epsilon" in params): @@ -810,6 +822,7 @@ class SVC(SVMBase, self.target_dtype = np.int64 self.probability = self._cpu_model.probability self.n_classes_ = self._cpu_model.n_classes_ + self.decision_function_shape = self._cpu_model.decision_function_shape def turn_cpu_into_gpu(cpu_est, params): gpu_est = SVC(**params) @@ -883,6 +896,8 @@ class SVC(SVMBase, elif self.decision_function_shape == 'ovo': gpu_est = OneVsOneClassifier(SVC) gpu_est.pairwise_indices_ = None + else: + raise ValueError gpu_est.classes_ = classes estimators = cpu_est.estimators_ gpu_est.estimators_ = [turn_cpu_into_gpu(est, params) for est in estimators] @@ -920,9 +935,12 @@ class SVC(SVMBase, self.multiclass_svc.multiclass_estimator.label_binarizer_.fit(self._cpu_model.classes_) elif strategy == 'ovo': self.multiclass_svc.multiclass_estimator.pairwise_indices_ = None + else: + raise ValueError def gpu_to_cpu(self): self._cpu_model.n_classes_ = self.n_classes_ + self._cpu_model.decision_function_shape = self.decision_function_shape def turn_gpu_into_cpu(gpu_est, params): cpu_est = skSVC(**params) @@ -957,6 +975,8 @@ class SVC(SVMBase, elif self.decision_function_shape == 'ovo': cpu_est = OneVsOneClassifier(skSVC) cpu_est.pairwise_indices_ = None + else: + raise ValueError cpu_est.classes_ = classes estimators = gpu_est.estimators_ cpu_est.estimators_ = [turn_gpu_into_cpu(est, params) for est in estimators] @@ -986,6 +1006,8 @@ class SVC(SVMBase, elif self.decision_function_shape == 'ovo': self._cpu_model.multi_class_model = OneVsOneClassifier(skSVC) self._cpu_model.multi_class_model.pairwise_indices_ = None + else: + raise ValueError self._cpu_model.multi_class_model.classes_ = classes params = self.get_params() diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index 63aeb0a694..cc5836227e 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -1063,7 +1063,10 @@ def test_svc_methods( decision_function_shape=decision_function_shape, ) ref_model.fit(X_train, y_train) - ref_output = ref_model.predict(X_test) + if probability: + ref_output = ref_model.predict_proba(X_test) + else: + ref_output = ref_model.predict(X_test) model = SVC( probability=probability, @@ -1072,10 +1075,21 @@ def test_svc_methods( with using_device_type(train_device): model.fit(X_train, y_train) with using_device_type(infer_device): - output = model.predict(X_test) - - correct_percentage = (ref_output == output).sum() / ref_output.size - assert correct_percentage > 0.9 + if probability: + output = model.predict_proba(X_test) + else: + output = model.predict(X_test) + + if probability: + eps = 0.25 + mismatches = ( + (output <= ref_output - eps) | (output >= ref_output + eps) + ).sum() + outlier_percentage = mismatches / ref_output.size + assert outlier_percentage < 0.03 + else: + correct_percentage = (ref_output == output).sum() / ref_output.size + assert correct_percentage > 0.9 @pytest.mark.parametrize("train_device", ["cpu", "gpu"]) From 8df9ac5cbc212199b60f5abe42d984643b210667 Mon Sep 17 00:00:00 2001 From: viclafargue Date: Wed, 11 Dec 2024 18:06:44 +0100 Subject: [PATCH 9/9] answered review --- docs/source/api.rst | 2 ++ python/cuml/cuml/svm/svc.pyx | 28 +++++++++++++------ .../cuml/cuml/tests/test_device_selection.py | 12 ++++---- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index a3a2ab73cc..c02feb394f 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -419,6 +419,8 @@ Support Vector Machines .. autoclass:: cuml.svm.SVR :members: +.. autoclass:: cuml.svm.SVC + .. autoclass:: cuml.svm.LinearSVC :members: diff --git a/python/cuml/cuml/svm/svc.pyx b/python/cuml/cuml/svm/svc.pyx index 1cdc8638ee..bd3e80adb0 100644 --- a/python/cuml/cuml/svm/svc.pyx +++ b/python/cuml/cuml/svm/svc.pyx @@ -326,6 +326,9 @@ class SVC(SVMBase, `_ while ``'ovr'`` selects `OneVsRestClassifier `_ + + .. versionadded:: 25.02 + The parameter `multiclass_strategy` was renamed to `decision_function_shape`. nochange_steps : int (default = 1000) We monitor how much our stopping criteria changes during outer iterations. If it does not change (changes less then 1e-3*tol) @@ -345,6 +348,14 @@ class SVC(SVMBase, verbose : int or boolean, default=False Sets logging level. It must be one of `cuml.common.logger.level_*`. See :ref:`verbosity-levels` for more info. + multiclass_strategy + Multiclass classification strategy. ``'ovo'`` uses `OneVsOneClassifier + `_ + while ``'ovr'`` selects `OneVsRestClassifier + `_ + + .. versionchanged:: 25.02 + Renamed to `decision_function_shape`. Will be removed in later versions. Attributes ---------- @@ -402,7 +413,7 @@ class SVC(SVMBase, max_iter=-1, nochange_steps=1000, verbose=False, output_type=None, probability=False, random_state=None, class_weight=None, decision_function_shape='ovo', - multiclass_strategy=None): + multiclass_strategy="warn"): super().__init__( handle=handle, C=C, @@ -424,15 +435,8 @@ class SVC(SVMBase, self.class_weight = class_weight self.svmType = C_SVC - if multiclass_strategy: - decision_function_shape = multiclass_strategy - warnings.simplefilter(action="always", category=FutureWarning) - warnings.warn('Parameter "multiclass_strategy" has been' - ' deprecated. Please use the' - ' "decision_function_shape" parameter instead.', - FutureWarning) - self.decision_function_shape = decision_function_shape + self.multiclass_strategy = multiclass_strategy @property @cuml.internals.api_base_return_array_skipall @@ -584,6 +588,12 @@ class SVC(SVMBase, Fit the model with X and y. """ + if self.multiclass_strategy != "warn": + self.decision_function_shape = self.multiclass_strategy + warnings.warn('Parameter "multiclass_strategy" has been' + ' deprecated. Please use the' + ' "decision_function_shape" parameter instead.', + FutureWarning) self.n_classes_ = self._get_num_classes(y) diff --git a/python/cuml/cuml/tests/test_device_selection.py b/python/cuml/cuml/tests/test_device_selection.py index cc5836227e..914deab5a1 100644 --- a/python/cuml/cuml/tests/test_device_selection.py +++ b/python/cuml/cuml/tests/test_device_selection.py @@ -146,7 +146,7 @@ def make_reg_dataset(): ) -def make_class_dataset(n_classes): +def make_classification_dataset(n_classes): X, y = make_classification( n_samples=2000, n_features=20, @@ -181,10 +181,12 @@ def make_blob_dataset(): X_train_reg, y_train_reg, X_test_reg = make_reg_dataset() -X_train_class, y_train_class, X_test_class = make_class_dataset(2) -X_train_multiclass, y_train_multiclass, X_test_multiclass = make_class_dataset( - 5 -) +X_train_class, y_train_class, X_test_class = make_classification_dataset(2) +( + X_train_multiclass, + y_train_multiclass, + X_test_multiclass, +) = make_classification_dataset(5) X_train_blob, y_train_blob, X_test_blob = make_blob_dataset()