"""Compatibility fixes for older version of the dependencies If you add content to this file, please give the version of the package at which the fix is no longer needed. """ # Authors: The scikit-learn developers # SPDX-License-Identifier: BSD-3-Clause import platform import struct import numpy as np import scipy import scipy.sparse.linalg import scipy.stats try: import pandas as pd except ImportError: pd = None from sklearn.externals._packaging.version import parse as parse_version from sklearn.utils.parallel import _get_threadpool_controller _IS_32BIT = 8 * struct.calcsize("P") == 32 _IS_WASM = platform.machine() in ["wasm32", "wasm64"] np_version = parse_version(np.__version__) np_base_version = parse_version(np_version.base_version) sp_version = parse_version(scipy.__version__) sp_base_version = parse_version(sp_version.base_version) # TODO: We can consider removing the containers and importing # directly from SciPy when sparse matrices will be deprecated. CSR_CONTAINERS = [scipy.sparse.csr_matrix, scipy.sparse.csr_array] CSC_CONTAINERS = [scipy.sparse.csc_matrix, scipy.sparse.csc_array] COO_CONTAINERS = [scipy.sparse.coo_matrix, scipy.sparse.coo_array] LIL_CONTAINERS = [scipy.sparse.lil_matrix, scipy.sparse.lil_array] DOK_CONTAINERS = [scipy.sparse.dok_matrix, scipy.sparse.dok_array] BSR_CONTAINERS = [scipy.sparse.bsr_matrix, scipy.sparse.bsr_array] DIA_CONTAINERS = [scipy.sparse.dia_matrix, scipy.sparse.dia_array] # Remove when minimum scipy version is 1.11.0 try: from scipy.sparse import sparray # noqa: F401 SPARRAY_PRESENT = True except ImportError: SPARRAY_PRESENT = False def _object_dtype_isnan(X): return X != X # TODO: Remove when SciPy 1.11 is the minimum supported version def _mode(a, axis=0): mode = scipy.stats.mode(a, axis=axis, keepdims=True) if sp_version >= parse_version("1.10.999"): # scipy.stats.mode has changed returned array shape with axis=None # and keepdims=True, see https://github.com/scipy/scipy/pull/17561 if axis is None: mode = np.ravel(mode) return mode # TODO: Remove when Scipy 1.12 is the minimum supported version # Use git grep to see where this is used and update them too. SCIPY_VERSION_BELOW_1_12 = sp_base_version < parse_version("1.12.0") # TODO: Remove when Scipy 1.15 is the minimum supported version # Use git grep to see where this is used and update them too. SCIPY_VERSION_BELOW_1_15 = sp_base_version < parse_version("1.15.0") # TODO: Remove when Scipy 1.12 is the minimum supported version if sp_base_version >= parse_version("1.12.0"): _sparse_linalg_cg = scipy.sparse.linalg.cg else: def _sparse_linalg_cg(A, b, **kwargs): if "rtol" in kwargs: kwargs["tol"] = kwargs.pop("rtol") if "atol" not in kwargs: kwargs["atol"] = "legacy" return scipy.sparse.linalg.cg(A, b, **kwargs) # TODO: Fuse the modern implementations of _sparse_min_max and _sparse_nan_min_max # into the public min_max_axis function when SciPy 1.11 is the minimum supported # version and delete the backport in the else branch below. if sp_base_version >= parse_version("1.11.0"): def _sparse_min_max(X, axis): the_min = X.min(axis=axis) the_max = X.max(axis=axis) if axis is not None: the_min = the_min.toarray().ravel() the_max = the_max.toarray().ravel() return the_min, the_max def _sparse_nan_min_max(X, axis): the_min = X.nanmin(axis=axis) the_max = X.nanmax(axis=axis) if axis is not None: the_min = the_min.toarray().ravel() the_max = the_max.toarray().ravel() return the_min, the_max else: # This code is mostly taken from scipy 0.14 and extended to handle nans, see # https://github.com/scikit-learn/scikit-learn/pull/11196 def _minor_reduce(X, ufunc): major_index = np.flatnonzero(np.diff(X.indptr)) # reduceat tries casts X.indptr to intp, which errors # if it is int64 on a 32 bit system. # Reinitializing prevents this where possible, see #13737 X = type(X)((X.data, X.indices, X.indptr), shape=X.shape) value = ufunc.reduceat(X.data, X.indptr[major_index]) return major_index, value def _min_or_max_axis(X, axis, min_or_max): N = X.shape[axis] if N == 0: raise ValueError("zero-size array to reduction operation") M = X.shape[1 - axis] mat = X.tocsc() if axis == 0 else X.tocsr() mat.sum_duplicates() major_index, value = _minor_reduce(mat, min_or_max) not_full = np.diff(mat.indptr)[major_index] < N value[not_full] = min_or_max(value[not_full], 0) mask = value != 0 major_index = np.compress(mask, major_index) value = np.compress(mask, value) if axis == 0: res = scipy.sparse.coo_array( (value, (np.zeros(len(value)), major_index)), dtype=X.dtype, shape=(1, M), ) else: res = scipy.sparse.coo_array( (value, (major_index, np.zeros(len(value)))), dtype=X.dtype, shape=(M, 1), ) return res.toarray().ravel() def _sparse_min_or_max(X, axis, min_or_max): if axis is None: if 0 in X.shape: raise ValueError("zero-size array to reduction operation") zero = X.dtype.type(0) if X.nnz == 0: return zero m = min_or_max.reduce(X.data.ravel()) if X.nnz != np.prod(X.shape): m = min_or_max(zero, m) return m if axis < 0: axis += 2 if (axis == 0) or (axis == 1): return _min_or_max_axis(X, axis, min_or_max) else: raise ValueError("invalid axis, use 0 for rows, or 1 for columns") def _sparse_min_max(X, axis): return ( _sparse_min_or_max(X, axis, np.minimum), _sparse_min_or_max(X, axis, np.maximum), ) def _sparse_nan_min_max(X, axis): return ( _sparse_min_or_max(X, axis, np.fmin), _sparse_min_or_max(X, axis, np.fmax), ) # For +1.25 NumPy versions exceptions and warnings are being moved # to a dedicated submodule. if np_version >= parse_version("1.25.0"): from numpy.exceptions import ComplexWarning, VisibleDeprecationWarning else: from numpy import ( # noqa: F401 ComplexWarning, VisibleDeprecationWarning, ) # TODO: Adapt when Pandas > 2.2 is the minimum supported version def pd_fillna(pd, frame): pd_version = parse_version(pd.__version__).base_version if parse_version(pd_version) < parse_version("2.2"): frame = frame.fillna(value=np.nan) else: infer_objects_kwargs = ( {} if parse_version(pd_version) >= parse_version("3") else {"copy": False} ) if parse_version(pd_version) < parse_version("3.0"): with pd.option_context("future.no_silent_downcasting", True): frame = frame.fillna(value=np.nan).infer_objects(**infer_objects_kwargs) else: frame = frame.fillna(value=np.nan).infer_objects(**infer_objects_kwargs) return frame # TODO: remove when SciPy 1.12 is the minimum supported version def _preserve_dia_indices_dtype( sparse_container, original_container_format, requested_sparse_format ): """Preserve indices dtype for SciPy < 1.12 when converting from DIA to CSR/CSC. For SciPy < 1.12, DIA arrays indices are upcasted to `np.int64` that is inconsistent with DIA matrices. We downcast the indices dtype to `np.int32` to be consistent with DIA matrices. The converted indices arrays are affected back inplace to the sparse container. Parameters ---------- sparse_container : sparse container Sparse container to be checked. requested_sparse_format : str or bool The type of format of `sparse_container`. Notes ----- See https://github.com/scipy/scipy/issues/19245 for more details. """ if original_container_format == "dia_array" and requested_sparse_format in ( "csr", "coo", ): if requested_sparse_format == "csr": index_dtype = _smallest_admissible_index_dtype( arrays=(sparse_container.indptr, sparse_container.indices), maxval=max(sparse_container.nnz, sparse_container.shape[1]), check_contents=True, ) sparse_container.indices = sparse_container.indices.astype( index_dtype, copy=False ) sparse_container.indptr = sparse_container.indptr.astype( index_dtype, copy=False ) else: # requested_sparse_format == "coo" index_dtype = _smallest_admissible_index_dtype( maxval=max(sparse_container.shape) ) sparse_container.row = sparse_container.row.astype(index_dtype, copy=False) sparse_container.col = sparse_container.col.astype(index_dtype, copy=False) # TODO: remove when SciPy 1.12 is the minimum supported version def _smallest_admissible_index_dtype(arrays=(), maxval=None, check_contents=False): """Based on input (integer) arrays `a`, determine a suitable index data type that can hold the data in the arrays. This function returns `np.int64` if it either required by `maxval` or based on the largest precision of the dtype of the arrays passed as argument, or by their contents (when `check_contents is True`). If none of the condition requires `np.int64` then this function returns `np.int32`. Parameters ---------- arrays : ndarray or tuple of ndarrays, default=() Input arrays whose types/contents to check. maxval : float, default=None Maximum value needed. check_contents : bool, default=False Whether to check the values in the arrays and not just their types. By default, check only the types. Returns ------- dtype : {np.int32, np.int64} Suitable index data type (int32 or int64). """ int32min = np.int32(np.iinfo(np.int32).min) int32max = np.int32(np.iinfo(np.int32).max) if maxval is not None: if maxval > np.iinfo(np.int64).max: raise ValueError( f"maxval={maxval} is to large to be represented as np.int64." ) if maxval > int32max: return np.int64 if isinstance(arrays, np.ndarray): arrays = (arrays,) for arr in arrays: if not isinstance(arr, np.ndarray): raise TypeError( f"Arrays should be of type np.ndarray, got {type(arr)} instead." ) if not np.issubdtype(arr.dtype, np.integer): raise ValueError( f"Array dtype {arr.dtype} is not supported for index dtype. We expect " "integral values." ) if not np.can_cast(arr.dtype, np.int32): if not check_contents: # when `check_contents` is False, we stay on the safe side and return # np.int64. return np.int64 if arr.size == 0: # a bigger type not needed yet, let's look at the next array continue else: maxval = arr.max() minval = arr.min() if minval < int32min or maxval > int32max: # a big index type is actually needed return np.int64 return np.int32 # TODO: Remove when SciPy 1.12 is the minimum supported version if sp_version < parse_version("1.12"): from sklearn.externals._scipy.sparse.csgraph import laplacian else: from scipy.sparse.csgraph import ( laplacian, # noqa: F401 # pragma: no cover ) # TODO: Remove when Python min version >= 3.12. def tarfile_extractall(tarfile, path): try: # Use filter="data" to prevent the most dangerous security issues. # For more details, see # https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall tarfile.extractall(path, filter="data") except TypeError: tarfile.extractall(path) def _in_unstable_openblas_configuration(): """Return True if in an unstable configuration for OpenBLAS""" # Import libraries which might load OpenBLAS. import numpy # noqa: F401 import scipy # noqa: F401 modules_info = _get_threadpool_controller().info() open_blas_used = any(info["internal_api"] == "openblas" for info in modules_info) if not open_blas_used: return False # OpenBLAS 0.3.16 fixed instability for arm64, see: # https://github.com/xianyi/OpenBLAS/blob/1b6db3dbba672b4f8af935bd43a1ff6cff4d20b7/Changelog.txt#L56-L58 openblas_arm64_stable_version = parse_version("0.3.16") for info in modules_info: if info["internal_api"] != "openblas": continue openblas_version = info.get("version") openblas_architecture = info.get("architecture") if openblas_version is None or openblas_architecture is None: # Cannot be sure that OpenBLAS is good enough. Assume unstable: return True # pragma: no cover if ( openblas_architecture == "neoversen1" and parse_version(openblas_version) < openblas_arm64_stable_version ): # See discussions in https://github.com/numpy/numpy/issues/19411 return True # pragma: no cover return False # TODO: Remove when Scipy 1.15 is the minimum supported version. In scipy 1.15, # the internal info details (via 'iprint' and 'disp' options) were dropped, # following the LBFGS rewrite from Fortran to C, see # https://github.com/scipy/scipy/issues/23186#issuecomment-2987801035. For # scipy 1.15, 'iprint' and 'disp' have no effect and for scipy >= 1.16 a # DeprecationWarning is emitted. def _get_additional_lbfgs_options_dict(key, value): return {} if sp_version >= parse_version("1.15") else {key: value} # TODO: Replace when Scipy 1.12 is the minimum supported version # fixes for transitioning scipy.sparse function names if not SCIPY_VERSION_BELOW_1_12: _sparse_eye_array = scipy.sparse.eye_array _sparse_diags_array = scipy.sparse.diags_array def _sparse_random_array( shape, *, density=0.01, format="coo", dtype=None, random_state=None, rng=None, data_sampler=None, ): X = scipy.sparse.random_array( shape, density=density, format=format, dtype=dtype, random_state=rng or random_state, data_sampler=data_sampler, ) _ensure_sparse_index_int32(X) return X else: def _sparse_eye_array(m, n=None, *, k=0, dtype=float, format=None): A = scipy.sparse.eye(m, n, k=k, dtype=dtype) return scipy.sparse.dia_array(A).asformat(format) def _sparse_diags_array( diagonals, /, *, offsets=0, shape=None, format=None, dtype=None ): A = scipy.sparse.diags(diagonals, offsets=offsets, shape=shape, dtype=dtype) return scipy.sparse.dia_array(A).asformat(format) def _sparse_random_array( shape, *, density=0.01, format="coo", dtype=None, random_state=None, rng=None, data_sampler=None, ): A = scipy.sparse.random( *shape, density=density, dtype=dtype, random_state=rng or random_state, data_rvs=data_sampler, ) return scipy.sparse.coo_array(A).asformat(format) # TODO: remove when SciPy 1.15 is minimal supported version # fix for casting index arrays def _ensure_sparse_index_int32(A): """Safely ensure that index arrays are int32.""" if A.format in ("csc", "csr", "bsr"): A.indices, A.indptr = _safely_cast_index_arrays(A) elif A.format == "coo": if hasattr(A, "coords"): A.coords = _safely_cast_index_arrays(A) elif hasattr(A, "indices"): A.indices = _safely_cast_index_arrays(A) else: A.row, A.col = _safely_cast_index_arrays(A) elif A.format == "dia": A.offsets = _safely_cast_index_arrays(A) # TODO: remove when SciPy 1.15 is minimal supported version # (based on scipy.sparse._sputils.py function with same name) def _safely_cast_index_arrays(A, idx_dtype=np.int32, msg=""): """Safely cast sparse array indices to `idx_dtype`. Check the shape of `A` to determine if it is safe to cast its index arrays to dtype `idx_dtype`. If any dimension in shape is larger than fits in the dtype, casting is unsafe so raise ``ValueError``. If safe, cast the index arrays to `idx_dtype` and return the result without changing the input `A`. The caller can assign results to `A` attributes if desired or use the recast index arrays directly. Unless downcasting is needed, the original index arrays are returned. You can test e.g. ``A.indptr is new_indptr`` to see if downcasting occurred. See SciPy: scipy.sparse._sputils.py for more info on safely_cast_index_arrays() """ max_value = np.iinfo(idx_dtype).max if A.format in ("csc", "csr"): if A.indptr[-1] > max_value: raise ValueError(f"indptr values too large for {msg}") # check shape vs dtype if max(*A.shape) > max_value: if (A.indices > max_value).any(): raise ValueError(f"indices values too large for {msg}") indices = A.indices.astype(idx_dtype, copy=False) indptr = A.indptr.astype(idx_dtype, copy=False) return indices, indptr elif A.format == "coo": coords = getattr(A, "coords", None) if coords is None: coords = getattr(A, "indices", None) if coords is None: coords = (A.row, A.col) if max(*A.shape) > max_value: if any((co > max_value).any() for co in coords): raise ValueError(f"coords values too large for {msg}") return tuple(co.astype(idx_dtype, copy=False) for co in coords) elif A.format == "dia": if max(*A.shape) > max_value: if (A.offsets > max_value).any(): raise ValueError(f"offsets values too large for {msg}") offsets = A.offsets.astype(idx_dtype, copy=False) return offsets elif A.format == "bsr": R, C = A.blocksize if A.indptr[-1] * R > max_value: raise ValueError("indptr values too large for {msg}") if max(*A.shape) > max_value: if (A.indices * C > max_value).any(): raise ValueError(f"indices values too large for {msg}") indices = A.indices.astype(idx_dtype, copy=False) indptr = A.indptr.astype(idx_dtype, copy=False) return indices, indptr # DOK and LIL formats are not associated with index arrays. # TODO remove when matplotlib 3.10 is the minimal supported version # and replace usage with `mpl.color_sequences['petroff10']` PETROFF_COLORS = [ "#3f90da", "#ffa90e", "#bd1f01", "#94a4a2", "#832db6", "#a96b59", "#e76300", "#b9ac70", "#717581", "#92dadd", ]