Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DEPR: fillna method kwd #53496

Merged
merged 14 commits into from
Jun 7, 2023
1 change: 1 addition & 0 deletions ci/code_checks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
pandas.core.groupby.SeriesGroupBy.cumprod \
pandas.core.groupby.SeriesGroupBy.cumsum \
pandas.core.groupby.SeriesGroupBy.diff \
pandas.core.groupby.SeriesGroupBy.fillna \
pandas.core.groupby.SeriesGroupBy.ffill \
pandas.core.groupby.SeriesGroupBy.max \
pandas.core.groupby.SeriesGroupBy.median \
Expand Down
2 changes: 1 addition & 1 deletion doc/source/getting_started/comparison/includes/missing.rst
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Forward fill from previous rows

.. ipython:: python

outer_join.fillna(method="ffill")
outer_join.ffill()

Replace missing values with a specified value
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Expand Down
4 changes: 2 additions & 2 deletions doc/source/user_guide/basics.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1377,12 +1377,12 @@ These methods require that the indexes are **ordered** increasing or
decreasing.

Note that the same result could have been achieved using
:ref:`fillna <missing_data.fillna>` (except for ``method='nearest'``) or
:ref:`ffill <missing_data.fillna>` (except for ``method='nearest'``) or
:ref:`interpolate <missing_data.interpolate>`:

.. ipython:: python

ts2.reindex(ts.index).fillna(method="ffill")
ts2.reindex(ts.index).ffill()

:meth:`~Series.reindex` will raise a ValueError if the index is not monotonically
increasing or decreasing. :meth:`~Series.fillna` and :meth:`~Series.interpolate`
Expand Down
6 changes: 3 additions & 3 deletions doc/source/user_guide/missing_data.rst
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ objects.
:suppress:

df = df2.loc[:, ["one", "two", "three"]]
a = df2.loc[df2.index[:5], ["one", "two"]].fillna(method="pad")
a = df2.loc[df2.index[:5], ["one", "two"]].ffill()
b = df2.loc[df2.index[:5], ["one", "two", "three"]]

.. ipython:: python
Expand Down Expand Up @@ -237,7 +237,7 @@ can propagate non-NA values forward or backward:
.. ipython:: python

df
df.fillna(method="pad")
df.ffill()

.. _missing_data.fillna.limit:

Expand All @@ -254,7 +254,7 @@ we can use the ``limit`` keyword:
.. ipython:: python

df
df.fillna(method="pad", limit=1)
df.ffill(limit=1)

To remind you, these are the available filling methods:

Expand Down
1 change: 1 addition & 0 deletions doc/source/whatsnew/v0.10.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ labeled the aggregated group with the end of the interval: the next day).
valid code. You must either specify a fill value or an interpolation method:

.. ipython:: python
:okwarning:

s = pd.Series([np.nan, 1.0, 2.0, np.nan, 4])
s
Expand Down
2 changes: 2 additions & 0 deletions doc/source/whatsnew/v2.1.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,8 @@ Deprecations
- Deprecated constructing :class:`SparseArray` from scalar data, pass a sequence instead (:issue:`53039`)
- Deprecated option "mode.use_inf_as_na", convert inf entries to ``NaN`` before instead (:issue:`51684`)
- Deprecated positional indexing on :class:`Series` with :meth:`Series.__getitem__` and :meth:`Series.__setitem__`, in a future version ``ser[item]`` will *always* interpret ``item`` as a label, not a position (:issue:`50617`)
- Deprecated the "method" and "limit" keywords on :meth:`Series.fillna`, :meth:`DataFrame.fillna`, :meth:`SeriesGroupBy.fillna`, :meth:`DataFrameGroupBy.fillna`, and :meth:`Resampler.fillna`, use ``obj.bfill()`` or ``obj.ffill()`` instead (:issue:`53394`)
-

.. ---------------------------------------------------------------------------
.. _whatsnew_210.performance:
Expand Down
12 changes: 12 additions & 0 deletions pandas/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,18 @@ def pytest_collection_modifyitems(items, config) -> None:
"first is deprecated and will be removed in a future version. "
"Please create a mask and filter using `.loc` instead",
),
(
"Resampler.fillna",
"DatetimeIndexResampler.fillna is deprecated",
),
(
"DataFrameGroupBy.fillna",
"DataFrameGroupBy.fillna with 'method' is deprecated",
),
(
"DataFrameGroupBy.fillna",
"DataFrame.fillna with 'method' is deprecated",
),
]

for item in items:
Expand Down
154 changes: 109 additions & 45 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -6803,6 +6803,61 @@ def convert_dtypes(
# ----------------------------------------------------------------------
# Filling NA's

def _deprecate_downcast(self, downcast) -> None:
if isinstance(downcast, dict):
# GH#40988
for dc in downcast.values():
if dc is not None and dc is not False and dc != "infer":
warnings.warn(
"downcast entries other than None, False, and 'infer' "
"are deprecated and will raise in a future version",
FutureWarning,
stacklevel=find_stack_level(),
)
elif downcast is not None and downcast is not False and downcast != "infer":
# GH#40988
warnings.warn(
"downcast other than None, False, and 'infer' are deprecated "
"and will raise in a future version",
FutureWarning,
stacklevel=find_stack_level(),
)

@final
def _fillna_with_method(
self,
method: Literal["ffill", "bfill", "pad", "backfill"],
*,
axis: None | Axis = None,
inplace: bool_t = False,
limit: None | int = None,
downcast: dict | None = None,
):
if axis is None:
axis = 0
axis = self._get_axis_number(axis)
method = clean_fill_method(method)

if not self._mgr.is_single_block and axis == 1:
if inplace:
raise NotImplementedError()
result = self.T._fillna_with_method(method=method, limit=limit).T

return result

new_mgr = self._mgr.interpolate(
method=method,
axis=axis,
limit=limit,
inplace=inplace,
downcast=downcast,
)
result = self._constructor(new_mgr)
if inplace:
return self._update_inplace(result)
else:
return result.__finalize__(self, method="fillna")

@overload
def fillna(
self,
Expand Down Expand Up @@ -6874,6 +6929,9 @@ def fillna(
* ffill: propagate last valid observation forward to next valid.
* backfill / bfill: use next valid observation to fill gap.

.. deprecated:: 2.1.0
Use ffill or bfill instead.

axis : {axes_single_arg}
Axis along which to fill missing values. For `Series`
this parameter is unused and defaults to 0.
Expand Down Expand Up @@ -6927,15 +6985,6 @@ def fillna(
2 0.0 0.0 0.0 0.0
3 0.0 3.0 0.0 4.0

We can also propagate non-null values forward or backward.

>>> df.fillna(method="ffill")
A B C D
0 NaN 2.0 NaN 0.0
1 3.0 4.0 NaN 1.0
2 3.0 4.0 NaN 1.0
3 3.0 3.0 NaN 4.0

Replace all NaN elements in column 'A', 'B', 'C', and 'D', with 0, 1,
2, and 3 respectively.

Expand Down Expand Up @@ -6971,42 +7020,29 @@ def fillna(
"""
inplace = validate_bool_kwarg(inplace, "inplace")
value, method = validate_fillna_kwargs(value, method)

if isinstance(downcast, dict):
# GH#40988
for dc in downcast.values():
if dc is not None and dc is not False and dc != "infer":
warnings.warn(
"downcast entries other than None, False, and 'infer' "
"are deprecated and will raise in a future version",
FutureWarning,
stacklevel=find_stack_level(),
)
elif downcast is not None and downcast is not False and downcast != "infer":
# GH#40988
if method is not None:
warnings.warn(
"downcast other than None, False, and 'infer' are deprecated "
"and will raise in a future version",
f"{type(self).__name__}.fillna with 'method' is deprecated and "
"will raise in a future version. Use obj.ffill() or obj.bfill() "
"instead.",
FutureWarning,
stacklevel=find_stack_level(),
)

self._deprecate_downcast(downcast)

# set the default here, so functions examining the signaure
# can detect if something was set (e.g. in groupby) (GH9221)
if axis is None:
axis = 0
axis = self._get_axis_number(axis)

if value is None:
if not self._mgr.is_single_block and axis == 1:
if inplace:
raise NotImplementedError()
result = self.T.fillna(method=method, limit=limit).T

return result

new_data = self._mgr.interpolate(
method=method,
return self._fillna_with_method(
# error: Argument 1 to "_fillna_with_method" of "NDFrame" has
# incompatible type "Optional[Literal['backfill', 'bfill', 'ffill',
# 'pad']]"; expected "Literal['ffill', 'bfill', 'pad', 'backfill']"
method, # type: ignore[arg-type]
axis=axis,
limit=limit,
inplace=inplace,
Expand Down Expand Up @@ -7111,7 +7147,10 @@ def fillna(
if axis == 1:
result = self.T.fillna(value=value, limit=limit).T

new_data = result
# error: Incompatible types in assignment (expression
# has type "Self", variable has type "Union[ArrayManager,
# SingleArrayManager, BlockManager, SingleBlockManager]")
new_data = result # type: ignore[assignment]
else:
new_data = self._mgr.fillna(
value=value, limit=limit, inplace=inplace, downcast=downcast
Expand Down Expand Up @@ -7180,6 +7219,25 @@ def ffill(

Examples
--------
>>> df = pd.DataFrame([[np.nan, 2, np.nan, 0],
... [3, 4, np.nan, 1],
... [np.nan, np.nan, np.nan, np.nan],
... [np.nan, 3, np.nan, 4]],
... columns=list("ABCD"))
>>> df
A B C D
0 NaN 2.0 NaN 0.0
1 3.0 4.0 NaN 1.0
2 NaN NaN NaN NaN
3 NaN 3.0 NaN 4.0

>>> df.ffill()
A B C D
0 NaN 2.0 NaN 0.0
1 3.0 4.0 NaN 1.0
2 3.0 4.0 NaN 1.0
3 3.0 3.0 NaN 4.0

>>> ser = pd.Series([1, np.NaN, 2, 3])
>>> ser.ffill()
0 1.0
Expand All @@ -7188,8 +7246,10 @@ def ffill(
3 3.0
dtype: float64
"""
return self.fillna(
method="ffill", axis=axis, inplace=inplace, limit=limit, downcast=downcast
self._deprecate_downcast(downcast)

return self._fillna_with_method(
"ffill", axis=axis, inplace=inplace, limit=limit, downcast=downcast
)

@final
Expand Down Expand Up @@ -7319,8 +7379,9 @@ def bfill(
2 4.0 7
3 4.0 7
"""
return self.fillna(
method="bfill", axis=axis, inplace=inplace, limit=limit, downcast=downcast
self._deprecate_downcast(downcast)
return self._fillna_with_method(
"bfill", axis=axis, inplace=inplace, limit=limit, downcast=downcast
)

@final
Expand Down Expand Up @@ -9840,8 +9901,8 @@ def _align_frame(
)

if method is not None:
left = left.fillna(method=method, axis=fill_axis, limit=limit)
right = right.fillna(method=method, axis=fill_axis, limit=limit)
left = left._fillna_with_method(method, axis=fill_axis, limit=limit)
right = right._fillna_with_method(method, axis=fill_axis, limit=limit)

return left, right, join_index

Expand Down Expand Up @@ -9916,8 +9977,13 @@ def _align_series(
# fill
fill_na = notna(fill_value) or (method is not None)
if fill_na:
left = left.fillna(fill_value, method=method, limit=limit, axis=fill_axis)
right = right.fillna(fill_value, method=method, limit=limit)
fill_value, method = validate_fillna_kwargs(fill_value, method)
if method is not None:
left = left._fillna_with_method(method, limit=limit, axis=fill_axis)
right = right._fillna_with_method(method, limit=limit)
else:
left = left.fillna(fill_value, limit=limit, axis=fill_axis)
right = right.fillna(fill_value, limit=limit)

return left, right, join_index

Expand Down Expand Up @@ -11283,9 +11349,7 @@ def pct_change(
if fill_method is None:
data = self
else:
_data = self.fillna(method=fill_method, axis=axis, limit=limit)
assert _data is not None # needed for mypy
data = _data
data = self._fillna_with_method(fill_method, axis=axis, limit=limit)

shifted = data.shift(periods=periods, freq=freq, axis=axis, **kwargs)
# Unsupported left operand type for / ("Self")
Expand Down
Loading