fix: Support Pandas future.infer_string=True in report generation (#1674)

ssiegel · web-flow · commit 45c47c7d514e · 2025-03-25T09:44:09.000-07:00
* fix: Support Pandas future.infer_string=True in report generation

Previously, report generation encountered issues when
`future.infer_string=True` was set. This resulted in multiple warnings
("FutureWarning: Dtype inference on a pandas object is deprecated") and
failures when string columns contained only empty strings
("AttributeError: 'StringDtype' object has no attribute
'pyarrow_dtype'").
This change resolves the issue by explicitly setting the dtype to
"object" for the relevant operations.

* refactor: Refactor pandas option usage for backward compatibility

- Introduce the `optional_option_context` helper to replace the direct
  use of `pd.option_context("future.no_silent_downcasting", True)`,
  ensuring compatibility with older pandas versions that lack this
  option.
- Update the `future.infer_string` test to run only on pandas &gt;= 2.1,
  where it is applicable.

* Add type annotations for optional_option_context

* Make the silent downcasting explicit
diff --git a/src/ydata_profiling/model/pandas/describe_categorical_pandas.py b/src/ydata_profiling/model/pandas/describe_categorical_pandas.py
@@ -20,7 +20,7 @@
 
 
 def get_character_counts_vc(vc: pd.Series) -> pd.Series:
-    series = pd.Series(vc.index, index=vc)
+    series = pd.Series(vc.index, index=vc, dtype=object)
     characters = series[series != ""].apply(list)
     characters = characters.explode()
 
@@ -170,7 +170,7 @@ def word_summary_vc(vc: pd.Series, stop_words: List[str] = []) -> dict:
     # TODO: configurable lowercase/punctuation etc.
     # TODO: remove punctuation in words
 
-    series = pd.Series(vc.index, index=vc)
+    series = pd.Series(vc.index, index=vc, dtype=object)
     word_lists = series.str.lower().str.split()
     words = word_lists.explode().str.strip(string.punctuation + string.whitespace)
     word_counts = pd.Series(words.index, index=words)
@@ -188,7 +188,7 @@ def word_summary_vc(vc: pd.Series, stop_words: List[str] = []) -> dict:
 
 
 def length_summary_vc(vc: pd.Series) -> dict:
-    series = pd.Series(vc.index, index=vc)
+    series = pd.Series(vc.index, index=vc, dtype=object)
     length = series.str.len()
     length_counts = pd.Series(length.index, index=length)
     length_counts = length_counts.groupby(level=0, sort=False).sum()
diff --git a/src/ydata_profiling/model/pandas/summary_pandas.py b/src/ydata_profiling/model/pandas/summary_pandas.py
@@ -10,6 +10,7 @@
 
 from ydata_profiling.config import Settings
 from ydata_profiling.model.typeset import ProfilingTypeSet
+from ydata_profiling.utils.compat import optional_option_context
 from ydata_profiling.utils.dataframe import sort_column_names
 
 BaseSummarizer: Any = "BaseSummarizer"  # type: ignore
@@ -38,7 +39,8 @@ def pandas_describe_1d(
     """
 
     # Make sure pd.NA is not in the series
-    series = series.fillna(np.nan)
+    with optional_option_context("future.no_silent_downcasting", True):
+        series = series.fillna(np.nan).infer_objects(copy=False)
 
     has_cast_type = _is_cast_type_defined(typeset, series.name)  # type:ignore
     cast_type = (
diff --git a/src/ydata_profiling/utils/compat.py b/src/ydata_profiling/utils/compat.py
@@ -1,6 +1,8 @@
 """Utility functions for (version) compatibility"""
+
+from contextlib import contextmanager
 from functools import lru_cache
-from typing import Tuple
+from typing import Generator, Tuple
 
 import pandas as pd
 
@@ -12,3 +14,18 @@ def pandas_version_info() -> Tuple[int, ...]:
     akin to `sys.version_info` for the Python version.
     """
     return tuple(int(s) for s in pd.__version__.split("."))
+
+
+@contextmanager
+def optional_option_context(
+    option_key: str, value: object
+) -> Generator[None, None, None]:
+    """
+    A context manager that sets an option only if it is available in the
+    current pandas version; otherwise, it is a no-op.
+    """
+    try:
+        with pd.option_context(option_key, value):
+            yield
+    except pd.errors.OptionError:
+        yield
diff --git a/tests/unit/test_pd_future_infer_string.py b/tests/unit/test_pd_future_infer_string.py
@@ -0,0 +1,25 @@
+import pandas as pd
+import pytest
+
+from ydata_profiling import ProfileReport
+from ydata_profiling.utils.compat import pandas_version_info
+
+
+@pytest.fixture()
+def df():
+    df = pd.DataFrame(
+        {
+            "foo": [1, 2, 3],
+            "bar": ["", "", ""],
+        }
+    )
+    return df
+
+
+@pytest.mark.skipif(
+    pandas_version_info() < (2, 1, 0), reason="requires pandas 2.1 or higher"
+)
+def test_pd_future_infer_string(df: pd.DataFrame):
+    with pd.option_context("future.infer_string", True):
+        profile_report = ProfileReport(df, title="Test Report", progress_bar=False)
+        assert len(profile_report.to_html()) > 0
