python/pyspark/testing/pandasutils.py - master - githubmirror/spark

python/pyspark/testing/pandasutils.py
687 строк26 KB
yangjie01
[SPARK-55986][PYTHON] Upgrade black to 26.3.1
16 мар 2026, 19:20
16 мар 2026, 19:20cbcee8c
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import functools
import shutil
import tempfile
import warnings
from contextlib import contextmanager
import decimal
from typing import Any, Union

try:
    from pyspark.sql.pandas.utils import require_minimum_pandas_version

    require_minimum_pandas_version()
    import pandas as pd
except ImportError:
    pass

try:
    from pyspark.sql.pandas.utils import require_minimum_pyarrow_version

    require_minimum_pyarrow_version()
    import pyarrow as pa
except ImportError:
    pass

try:
    from pyspark.sql.pandas.utils import require_minimum_numpy_version

    require_minimum_numpy_version()
    import numpy as np
except ImportError:
    pass

from pyspark.loose_version import LooseVersion
import pyspark.pandas as ps
from pyspark.pandas.frame import DataFrame
from pyspark.pandas.indexes import Index
from pyspark.pandas.series import Series
from pyspark.pandas.utils import SPARK_CONF_ARROW_ENABLED
from pyspark.testing.sqlutils import ReusedSQLTestCase
from pyspark.testing.utils import is_ansi_mode_test
from pyspark.errors import PySparkAssertionError


def _assert_pandas_equal(
    left: Union[pd.DataFrame, pd.Series, pd.Index],
    right: Union[pd.DataFrame, pd.Series, pd.Index],
    checkExact: bool,
):
    from pandas.testing import assert_frame_equal, assert_index_equal, assert_series_equal

    if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
        try:
            kwargs = dict(check_freq=False)

            assert_frame_equal(
                left,
                right,
                check_index_type=("equiv" if len(left.index) > 0 else False),
                check_column_type=("equiv" if len(left.columns) > 0 else False),
                check_exact=checkExact,
                **kwargs,
            )
        except AssertionError:
            raise PySparkAssertionError(
                errorClass="DIFFERENT_PANDAS_DATAFRAME",
                messageParameters={
                    "left": left.to_string(),
                    "left_dtype": str(left.dtypes),
                    "right": right.to_string(),
                    "right_dtype": str(right.dtypes),
                },
            )
    elif isinstance(left, pd.Series) and isinstance(right, pd.Series):
        try:
            kwargs = dict(check_freq=False)
            assert_series_equal(
                left,
                right,
                check_index_type=("equiv" if len(left.index) > 0 else False),
                check_exact=checkExact,
                **kwargs,
            )
        except AssertionError:
            raise PySparkAssertionError(
                errorClass="DIFFERENT_PANDAS_SERIES",
                messageParameters={
                    "left": left.to_string(),
                    "left_dtype": str(left.dtype),
                    "right": right.to_string(),
                    "right_dtype": str(right.dtype),
                },
            )
    elif isinstance(left, pd.Index) and isinstance(right, pd.Index):
        try:
            assert_index_equal(left, right, check_exact=checkExact)
        except AssertionError:
            raise PySparkAssertionError(
                errorClass="DIFFERENT_PANDAS_INDEX",
                messageParameters={
                    "left": left,
                    "left_dtype": str(left.dtype),
                    "right": right,
                    "right_dtype": str(right.dtype),
                },
            )
    else:
        raise ValueError("Unexpected values: (%s, %s)" % (left, right))


def _assert_pandas_almost_equal(
    left: Union[pd.DataFrame, pd.Series, pd.Index],
    right: Union[pd.DataFrame, pd.Series, pd.Index],
    rtol: float = 1e-5,
    atol: float = 1e-8,
):
    """
    This function checks if given pandas objects approximately same,
    which means the conditions below:
      - Both objects are nullable
      - Compare decimals and floats, where two values a and b are approximately equal
        if they satisfy the following formula:
        absolute(a - b) <= (atol + rtol * absolute(b))
        where rtol=1e-5 and atol=1e-8 by default
    """

    def compare_vals_approx(val1, val2):
        if isinstance(val1, np.ndarray):
            return compare_vals_approx(list(val1), val2)
        if isinstance(val2, np.ndarray):
            return compare_vals_approx(val1, list(val2))
        # compare vals for approximate equality
        if isinstance(val1, (float, decimal.Decimal)) or isinstance(val2, (float, decimal.Decimal)):
            if abs(float(val1) - float(val2)) > (atol + rtol * abs(float(val2))):
                return False
        elif val1 != val2:
            return False
        return True

    if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
        if left.shape != right.shape:
            raise PySparkAssertionError(
                errorClass="DIFFERENT_PANDAS_DATAFRAME",
                messageParameters={
                    "left": left.to_string(),
                    "left_dtype": str(left.dtypes),
                    "right": right.to_string(),
                    "right_dtype": str(right.dtypes),
                },
            )
        for lcol, rcol in zip(left.columns, right.columns):
            if lcol != rcol:
                raise PySparkAssertionError(
                    errorClass="DIFFERENT_PANDAS_DATAFRAME",
                    messageParameters={
                        "left": left.to_string(),
                        "left_dtype": str(left.dtypes),
                        "right": right.to_string(),
                        "right_dtype": str(right.dtypes),
                    },
                )
            for lnull, rnull in zip(left[lcol].isnull(), right[rcol].isnull()):
                if lnull != rnull:
                    raise PySparkAssertionError(
                        errorClass="DIFFERENT_PANDAS_DATAFRAME",
                        messageParameters={
                            "left": left.to_string(),
                            "left_dtype": str(left.dtypes),
                            "right": right.to_string(),
                            "right_dtype": str(right.dtypes),
                        },
                    )
            for lval, rval in zip(left[lcol].dropna(), right[rcol].dropna()):
                if not compare_vals_approx(lval, rval):
                    raise PySparkAssertionError(
                        errorClass="DIFFERENT_PANDAS_DATAFRAME",
                        messageParameters={
                            "left": left.to_string(),
                            "left_dtype": str(left.dtypes),
                            "right": right.to_string(),
                            "right_dtype": str(right.dtypes),
                        },
                    )
        if left.columns.names != right.columns.names:
            raise PySparkAssertionError(
                errorClass="DIFFERENT_PANDAS_DATAFRAME",
                messageParameters={
                    "left": left.to_string(),
                    "left_dtype": str(left.dtypes),
                    "right": right.to_string(),
                    "right_dtype": str(right.dtypes),
                },
            )
    elif isinstance(left, pd.Series) and isinstance(right, pd.Series):
        if left.name != right.name or len(left) != len(right):
            raise PySparkAssertionError(
                errorClass="DIFFERENT_PANDAS_SERIES",
                messageParameters={
                    "left": left.to_string(),
                    "left_dtype": str(left.dtype),
                    "right": right.to_string(),
                    "right_dtype": str(right.dtype),
                },
            )
        for lnull, rnull in zip(left.isnull(), right.isnull()):
            if lnull != rnull:
                raise PySparkAssertionError(
                    errorClass="DIFFERENT_PANDAS_SERIES",
                    messageParameters={
                        "left": left.to_string(),
                        "left_dtype": str(left.dtype),
                        "right": right.to_string(),
                        "right_dtype": str(right.dtype),
                    },
                )
        for lval, rval in zip(left.dropna(), right.dropna()):
            if not compare_vals_approx(lval, rval):
                raise PySparkAssertionError(
                    errorClass="DIFFERENT_PANDAS_SERIES",
                    messageParameters={
                        "left": left.to_string(),
                        "left_dtype": str(left.dtype),
                        "right": right.to_string(),
                        "right_dtype": str(right.dtype),
                    },
                )
    elif isinstance(left, pd.MultiIndex) and isinstance(right, pd.MultiIndex):
        if len(left) != len(right):
            raise PySparkAssertionError(
                errorClass="DIFFERENT_PANDAS_MULTIINDEX",
                messageParameters={
                    "left": left,
                    "left_dtype": str(left.dtype),
                    "right": right,
                    "right_dtype": str(right.dtype),
                },
            )
        for lval, rval in zip(left, right):
            if not compare_vals_approx(lval, rval):
                raise PySparkAssertionError(
                    errorClass="DIFFERENT_PANDAS_MULTIINDEX",
                    messageParameters={
                        "left": left,
                        "left_dtype": str(left.dtype),
                        "right": right,
                        "right_dtype": str(right.dtype),
                    },
                )
    elif isinstance(left, pd.Index) and isinstance(right, pd.Index):
        if len(left) != len(right):
            raise PySparkAssertionError(
                errorClass="DIFFERENT_PANDAS_INDEX",
                messageParameters={
                    "left": left,
                    "left_dtype": str(left.dtype),
                    "right": right,
                    "right_dtype": str(right.dtype),
                },
            )
        for lnull, rnull in zip(left.isnull(), right.isnull()):
            if lnull != rnull:
                raise PySparkAssertionError(
                    errorClass="DIFFERENT_PANDAS_INDEX",
                    messageParameters={
                        "left": left,
                        "left_dtype": str(left.dtype),
                        "right": right,
                        "right_dtype": str(right.dtype),
                    },
                )
        for lval, rval in zip(left.dropna(), right.dropna()):
            if not compare_vals_approx(lval, rval):
                raise PySparkAssertionError(
                    errorClass="DIFFERENT_PANDAS_INDEX",
                    messageParameters={
                        "left": left,
                        "left_dtype": str(left.dtype),
                        "right": right,
                        "right_dtype": str(right.dtype),
                    },
                )
    else:
        if not isinstance(left, (pd.DataFrame, pd.Series, pd.Index)):
            raise PySparkAssertionError(
                errorClass="INVALID_TYPE_DF_EQUALITY_ARG",
                messageParameters={
                    "expected_type": f"{pd.DataFrame.__name__}, "
                    f"{pd.Series.__name__}, "
                    f"{pd.Index.__name__}, ",
                    "arg_name": "left",
                    "actual_type": type(left),
                },
            )
        elif not isinstance(right, (pd.DataFrame, pd.Series, pd.Index)):
            raise PySparkAssertionError(
                errorClass="INVALID_TYPE_DF_EQUALITY_ARG",
                messageParameters={
                    "expected_type": f"{pd.DataFrame.__name__}, "
                    f"{pd.Series.__name__}, "
                    f"{pd.Index.__name__}, ",
                    "arg_name": "right",
                    "actual_type": type(right),
                },
            )


class PandasOnSparkTestUtils:
    def convert_str_to_lambda(self, func: str):
        """
        This function converts `func` str to lambda call
        """
        return lambda x: getattr(x, func)()

    def sort_index_with_values(self, pobj: Any):
        assert isinstance(pobj, (pd.Series, pd.DataFrame, ps.Series, ps.DataFrame))

        if isinstance(pobj, (ps.Series, ps.DataFrame)):
            if isinstance(pobj, ps.Series):
                psdf = pobj._psdf[[pobj.name]]
            else:
                psdf = pobj
            scols = psdf._internal.index_spark_columns + psdf._internal.data_spark_columns
            sorted = psdf._sort(
                by=scols,
                ascending=True,
                na_position="last",
            )
            if isinstance(pobj, ps.Series):
                from pyspark.pandas.series import first_series

                return first_series(sorted)
            else:
                return sorted
        else:
            # quick-sort values and then stable-sort index
            if isinstance(pobj, pd.Series):
                return pobj.sort_values(
                    ascending=True,
                    na_position="last",
                ).sort_index(
                    ascending=True,
                    na_position="last",
                    kind="mergesort",
                )
            else:
                return pobj.sort_values(
                    by=list(pobj.columns),
                    ascending=True,
                    na_position="last",
                ).sort_index(
                    ascending=True,
                    na_position="last",
                    kind="mergesort",
                )

    def assertPandasEqual(self, left: Any, right: Any, check_exact: bool = True):
        _assert_pandas_equal(left, right, check_exact)

    def assertPandasAlmostEqual(
        self,
        left: Any,
        right: Any,
        rtol: float = 1e-5,
        atol: float = 1e-8,
    ):
        _assert_pandas_almost_equal(left, right, rtol=rtol, atol=atol)

    def assert_eq(
        self,
        left: Any,
        right: Any,
        check_exact: bool = True,
        almost: bool = False,
        rtol: float = 1e-5,
        atol: float = 1e-8,
        check_row_order: bool = True,
        ignore_null: bool = False,
    ):
        """
        Asserts if two arbitrary objects are equal or not. If given objects are
        Pandas-on-Spark DataFrame or Series, they are converted into pandas' and compared.

        :param left: object to compare
        :param right: object to compare
        :param check_exact: if this is False, the comparison is done less precisely.
        :param almost: if this is enabled, the comparison asserts approximate equality
            for float and decimal values, where two values a and b are approximately equal
            if they satisfy the following formula:
            absolute(a - b) <= (atol + rtol * absolute(b))
        :param rtol: The relative tolerance, used in asserting approximate equality for
            float values. Set to 1e-5 by default.
        :param atol: The absolute tolerance, used in asserting approximate equality for
            float values in actual and expected. Set to 1e-8 by default.
        :param check_row_order: A flag indicating whether the order of rows should be considered
            in the comparison. If set to False, row order will be ignored.
        :param ignore_null: if this is enabled, the comparison will ignore null values.
        """
        import pandas as pd
        from pandas.api.types import is_list_like

        if ignore_null:
            # We use _assert_pandas_almost_equal with atol=0 and rtol=0 to check if the
            # values are equal because null values are properly handled by it
            if not almost:
                # It's possible to set almost=True and ignore_null=True. In that case,
                # honor atol and rtol settings. ignore_null=True is implied by almost=True.
                almost = True
                rtol = 0
                atol = 0

        # for pandas-on-Spark DataFrames, allow choice to ignore row order
        if isinstance(left, (ps.DataFrame, ps.Series, ps.Index)):
            if left is None and right is None:
                return True
            elif left is None or right is None:
                return False

            if not isinstance(left, (DataFrame, Series, Index)):
                raise PySparkAssertionError(
                    errorClass="INVALID_TYPE_DF_EQUALITY_ARG",
                    messageParameters={
                        "expected_type": f"{DataFrame.__name__}, {Series.__name__}, "
                        f"{Index.__name__}",
                        "arg_name": "actual",
                        "actual_type": type(left),
                    },
                )
            elif not isinstance(
                right, (DataFrame, pd.DataFrame, Series, pd.Series, Index, pd.Index)
            ):
                raise PySparkAssertionError(
                    errorClass="INVALID_TYPE_DF_EQUALITY_ARG",
                    messageParameters={
                        "expected_type": f"{DataFrame.__name__}, "
                        f"{pd.DataFrame.__name__}, "
                        f"{Series.__name__}, "
                        f"{pd.Series.__name__}, "
                        f"{Index.__name__}"
                        f"{pd.Index.__name__}, ",
                        "arg_name": "expected",
                        "actual_type": type(right),
                    },
                )
            else:
                if not isinstance(left, (pd.DataFrame, pd.Index, pd.Series)):
                    left = self._ignore_arrow_dtypes(left.to_pandas())
                if not isinstance(right, (pd.DataFrame, pd.Index, pd.Series)):
                    right = self._ignore_arrow_dtypes(right.to_pandas())

                if not check_row_order:
                    if isinstance(left, pd.DataFrame) and len(left.columns) > 0:
                        left = left.sort_values(by=left.columns[0], ignore_index=True)
                    if isinstance(right, pd.DataFrame) and len(right.columns) > 0:
                        right = right.sort_values(by=right.columns[0], ignore_index=True)

                if almost:
                    _assert_pandas_almost_equal(left, right, rtol=rtol, atol=atol)
                else:
                    _assert_pandas_equal(left, right, checkExact=check_exact)

        lobj = self._ignore_arrow_dtypes(self._to_pandas(left))
        robj = self._ignore_arrow_dtypes(self._to_pandas(right))
        if isinstance(lobj, (pd.DataFrame, pd.Series, pd.Index)):
            if almost:
                _assert_pandas_almost_equal(lobj, robj, rtol=rtol, atol=atol)
            else:
                _assert_pandas_equal(lobj, robj, checkExact=check_exact)
        elif is_list_like(lobj) and is_list_like(robj):
            self.assertTrue(len(left) == len(right))
            for litem, ritem in zip(left, right):
                self.assert_eq(
                    litem, ritem, check_exact=check_exact, almost=almost, ignore_null=ignore_null
                )
        elif (lobj is not None and pd.isna(lobj)) and (robj is not None and pd.isna(robj)):
            pass
        else:
            if almost:
                self.assertAlmostEqual(lobj, robj)
            else:
                self.assertEqual(lobj, robj)

    @staticmethod
    def _to_pandas(obj: Any):
        if isinstance(obj, (DataFrame, Series, Index)):
            return obj.to_pandas()
        else:
            return obj

    @staticmethod
    def _ignore_arrow_dtypes(obj: Any):
        if LooseVersion(pd.__version__) < "3.0.0":
            return obj
        else:
            if isinstance(obj, pd.DataFrame):
                arrow_boolean_columns = [
                    name
                    for name, col in obj.items()
                    if isinstance(col.dtype, pd.ArrowDtype)
                    and col.dtype.pyarrow_dtype == pa.bool_()
                ]
                if arrow_boolean_columns:
                    return obj.astype({name: "boolean" for name in arrow_boolean_columns})
            elif isinstance(obj, (pd.Series, pd.Index)):
                if isinstance(obj.dtype, pd.ArrowDtype) and obj.dtype.pyarrow_dtype == pa.bool_():
                    return obj.astype("boolean")
            return obj


class PandasOnSparkTestCase(ReusedSQLTestCase, PandasOnSparkTestUtils):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.spark.conf.set(SPARK_CONF_ARROW_ENABLED, True)

    def setUp(self):
        super().setUp()
        self.assertEqual(is_ansi_mode_test, self.spark.conf.get("spark.sql.ansi.enabled") == "true")

    def tearDown(self):
        try:
            self.assertEqual(
                is_ansi_mode_test, self.spark.conf.get("spark.sql.ansi.enabled") == "true"
            )
        finally:
            super().tearDown()


class TestUtils:
    @contextmanager
    def temp_dir(self):
        tmp = tempfile.mkdtemp()
        try:
            yield tmp
        finally:
            shutil.rmtree(tmp)

    @contextmanager
    def temp_file(self):
        with self.temp_dir() as tmp:
            yield tempfile.mkstemp(dir=tmp)[1]


class ComparisonTestBase(PandasOnSparkTestCase):
    @property
    def psdf(self):
        return ps.from_pandas(self.pdf)

    @property
    def pdf(self):
        return self.psdf.to_pandas()


def compare_both(f=None, almost=True):
    if f is None:
        return functools.partial(compare_both, almost=almost)
    elif isinstance(f, bool):
        return functools.partial(compare_both, almost=f)

    @functools.wraps(f)
    def wrapped(self):
        if almost:
            compare = self.assertPandasAlmostEqual
        else:
            compare = self.assertPandasEqual

        for result_pandas, result_spark in zip(f(self, self.pdf), f(self, self.psdf)):
            compare(result_pandas, result_spark.to_pandas())

    return wrapped


@contextmanager
def assert_produces_warning(
    expected_warning=Warning,
    filter_level="always",
    check_stacklevel=True,
    raise_on_extra_warnings=True,
):
    """
    Context manager for running code expected to either raise a specific
    warning, or not raise any warnings. Verifies that the code raises the
    expected warning, and that it does not raise any other unexpected
    warnings. It is basically a wrapper around ``warnings.catch_warnings``.

    Notes
    -----
    Replicated from pandas/_testing/_warnings.py.

    Parameters
    ----------
    expected_warning : {Warning, False, None}, default Warning
        The type of Exception raised. ``exception.Warning`` is the base
        class for all warnings. To check that no warning is returned,
        specify ``False`` or ``None``.
    filter_level : str or None, default "always"
        Specifies whether warnings are ignored, displayed, or turned
        into errors.
        Valid values are:
        * "error" - turns matching warnings into exceptions
        * "ignore" - discard the warning
        * "always" - always emit a warning
        * "default" - print the warning the first time it is generated
          from each location
        * "module" - print the warning the first time it is generated
          from each module
        * "once" - print the warning the first time it is generated
    check_stacklevel : bool, default True
        If True, displays the line that called the function containing
        the warning to show were the function is called. Otherwise, the
        line that implements the function is displayed.
    raise_on_extra_warnings : bool, default True
        Whether extra warnings not of the type `expected_warning` should
        cause the test to fail.

    Examples
    --------
    >>> import warnings
    >>> with assert_produces_warning():
    ...     warnings.warn(UserWarning())
    ...
    >>> with assert_produces_warning(False): # doctest: +SKIP
    ...     warnings.warn(RuntimeWarning())
    ...
    Traceback (most recent call last):
        ...
    AssertionError: Caused unexpected warning(s): ['RuntimeWarning'].
    >>> with assert_produces_warning(UserWarning): # doctest: +SKIP
    ...     warnings.warn(RuntimeWarning())
    Traceback (most recent call last):
        ...
    AssertionError: Did not see expected warning of class 'UserWarning'
    ..warn:: This is *not* thread-safe.
    """
    __tracebackhide__ = True

    with warnings.catch_warnings(record=True) as w:
        saw_warning = False
        warnings.simplefilter(filter_level)
        yield w
        extra_warnings = []

        for actual_warning in w:
            if expected_warning and issubclass(actual_warning.category, expected_warning):
                saw_warning = True

                if check_stacklevel and issubclass(
                    actual_warning.category, (FutureWarning, DeprecationWarning)
                ):
                    from inspect import getframeinfo, stack

                    caller = getframeinfo(stack()[2][0])
                    msg = (
                        "Warning not set with correct stacklevel. ",
                        "File where warning is raised: {} != ".format(actual_warning.filename),
                        "{}. Warning message: {}".format(caller.filename, actual_warning.message),
                    )
                    assert actual_warning.filename == caller.filename, msg
            else:
                extra_warnings.append(
                    (
                        actual_warning.category.__name__,
                        actual_warning.message,
                        actual_warning.filename,
                        actual_warning.lineno,
                    )
                )
        if expected_warning:
            msg = "Did not see expected warning of class {}".format(repr(expected_warning.__name__))
            assert saw_warning, msg
        if raise_on_extra_warnings and extra_warnings:
            raise AssertionError("Caused unexpected warning(s): {}".format(repr(extra_warnings)))