From 38c2877c79e8625ea9c76f3af0a13e0764ebe09d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 19 Dec 2023 15:39:30 -0800 Subject: [PATCH] DEPR: keep_date_col, nested parse_dates in read_csv (#56569) * DEPR: keep_date_col, nested parse_dates in read_csv * update doc, mypy fixup --- doc/source/user_guide/io.rst | 8 + doc/source/whatsnew/v2.2.0.rst | 1 + pandas/io/parsers/readers.py | 76 ++++++-- pandas/tests/io/parser/test_parse_dates.py | 162 +++++++++++++----- .../io/parser/usecols/test_parse_dates.py | 42 ++++- 5 files changed, 231 insertions(+), 58 deletions(-) diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst index 863a663fc2413..843144c3453c2 100644 --- a/doc/source/user_guide/io.rst +++ b/doc/source/user_guide/io.rst @@ -836,6 +836,7 @@ order) and the new column names will be the concatenation of the component column names: .. ipython:: python + :okwarning: data = ( "KORD,19990127, 19:00:00, 18:56:00, 0.8100\n" @@ -856,6 +857,7 @@ By default the parser removes the component date columns, but you can choose to retain them via the ``keep_date_col`` keyword: .. ipython:: python + :okwarning: df = pd.read_csv( "tmp.csv", header=None, parse_dates=[[1, 2], [1, 3]], keep_date_col=True @@ -871,6 +873,7 @@ single column. You can also use a dict to specify custom name columns: .. ipython:: python + :okwarning: date_spec = {"nominal": [1, 2], "actual": [1, 3]} df = pd.read_csv("tmp.csv", header=None, parse_dates=date_spec) @@ -883,6 +886,7 @@ data columns: .. ipython:: python + :okwarning: date_spec = {"nominal": [1, 2], "actual": [1, 3]} df = pd.read_csv( @@ -902,6 +906,10 @@ data columns: for your data to store datetimes in this format, load times will be significantly faster, ~20x has been observed. +.. deprecated:: 2.2.0 + Combining date columns inside read_csv is deprecated. Use ``pd.to_datetime`` + on the relevant result columns instead. + Date parsing functions ++++++++++++++++++++++ diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst index 68c4ef34c8f4f..8c475791df64d 100644 --- a/doc/source/whatsnew/v2.2.0.rst +++ b/doc/source/whatsnew/v2.2.0.rst @@ -481,6 +481,7 @@ Other Deprecations - Deprecated strings ``H``, ``S``, ``U``, and ``N`` denoting units in :func:`to_timedelta` (:issue:`52536`) - Deprecated strings ``H``, ``T``, ``S``, ``L``, ``U``, and ``N`` denoting units in :class:`Timedelta` (:issue:`52536`) - Deprecated strings ``T``, ``S``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Second`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`52536`) +- Deprecated support for combining parsed datetime columns in :func:`read_csv` along with the ``keep_date_col`` keyword (:issue:`55569`) - Deprecated the :attr:`.DataFrameGroupBy.grouper` and :attr:`SeriesGroupBy.grouper`; these attributes will be removed in a future version of pandas (:issue:`56521`) - Deprecated the :class:`.Grouping` attributes ``group_index``, ``result_index``, and ``group_arraylike``; these will be removed in a future version of pandas (:issue:`56148`) - Deprecated the ``errors="ignore"`` option in :func:`to_datetime`, :func:`to_timedelta`, and :func:`to_numeric`; explicitly catch exceptions instead (:issue:`54467`) diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py index 61e8862d85288..4e19f401af229 100644 --- a/pandas/io/parsers/readers.py +++ b/pandas/io/parsers/readers.py @@ -41,6 +41,7 @@ from pandas.core.dtypes.common import ( is_file_like, is_float, + is_hashable, is_integer, is_list_like, pandas_dtype, @@ -649,7 +650,7 @@ def read_csv( skip_blank_lines: bool = ..., parse_dates: bool | Sequence[Hashable] | None = ..., infer_datetime_format: bool | lib.NoDefault = ..., - keep_date_col: bool = ..., + keep_date_col: bool | lib.NoDefault = ..., date_parser: Callable | lib.NoDefault = ..., date_format: str | dict[Hashable, str] | None = ..., dayfirst: bool = ..., @@ -709,7 +710,7 @@ def read_csv( skip_blank_lines: bool = ..., parse_dates: bool | Sequence[Hashable] | None = ..., infer_datetime_format: bool | lib.NoDefault = ..., - keep_date_col: bool = ..., + keep_date_col: bool | lib.NoDefault = ..., date_parser: Callable | lib.NoDefault = ..., date_format: str | dict[Hashable, str] | None = ..., dayfirst: bool = ..., @@ -769,7 +770,7 @@ def read_csv( skip_blank_lines: bool = ..., parse_dates: bool | Sequence[Hashable] | None = ..., infer_datetime_format: bool | lib.NoDefault = ..., - keep_date_col: bool = ..., + keep_date_col: bool | lib.NoDefault = ..., date_parser: Callable | lib.NoDefault = ..., date_format: str | dict[Hashable, str] | None = ..., dayfirst: bool = ..., @@ -829,7 +830,7 @@ def read_csv( skip_blank_lines: bool = ..., parse_dates: bool | Sequence[Hashable] | None = ..., infer_datetime_format: bool | lib.NoDefault = ..., - keep_date_col: bool = ..., + keep_date_col: bool | lib.NoDefault = ..., date_parser: Callable | lib.NoDefault = ..., date_format: str | dict[Hashable, str] | None = ..., dayfirst: bool = ..., @@ -903,7 +904,7 @@ def read_csv( # Datetime Handling parse_dates: bool | Sequence[Hashable] | None = None, infer_datetime_format: bool | lib.NoDefault = lib.no_default, - keep_date_col: bool = False, + keep_date_col: bool | lib.NoDefault = lib.no_default, date_parser: Callable | lib.NoDefault = lib.no_default, date_format: str | dict[Hashable, str] | None = None, dayfirst: bool = False, @@ -934,6 +935,38 @@ def read_csv( storage_options: StorageOptions | None = None, dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default, ) -> DataFrame | TextFileReader: + if keep_date_col is not lib.no_default: + # GH#55569 + warnings.warn( + "The 'keep_date_col' keyword in pd.read_csv is deprecated and " + "will be removed in a future version. Explicitly remove unwanted " + "columns after parsing instead.", + FutureWarning, + stacklevel=find_stack_level(), + ) + else: + keep_date_col = False + + if lib.is_list_like(parse_dates): + # GH#55569 + depr = False + # error: Item "bool" of "bool | Sequence[Hashable] | None" has no + # attribute "__iter__" (not iterable) + if not all(is_hashable(x) for x in parse_dates): # type: ignore[union-attr] + depr = True + elif isinstance(parse_dates, dict) and any( + lib.is_list_like(x) for x in parse_dates.values() + ): + depr = True + if depr: + warnings.warn( + "Support for nested sequences for 'parse_dates' in pd.read_csv " + "is deprecated. Combine the desired columns with pd.to_datetime " + "after parsing instead.", + FutureWarning, + stacklevel=find_stack_level(), + ) + if infer_datetime_format is not lib.no_default: warnings.warn( "The argument 'infer_datetime_format' is deprecated and will " @@ -1004,7 +1037,7 @@ def read_table( skip_blank_lines: bool = ..., parse_dates: bool | Sequence[Hashable] = ..., infer_datetime_format: bool | lib.NoDefault = ..., - keep_date_col: bool = ..., + keep_date_col: bool | lib.NoDefault = ..., date_parser: Callable | lib.NoDefault = ..., date_format: str | dict[Hashable, str] | None = ..., dayfirst: bool = ..., @@ -1061,7 +1094,7 @@ def read_table( skip_blank_lines: bool = ..., parse_dates: bool | Sequence[Hashable] = ..., infer_datetime_format: bool | lib.NoDefault = ..., - keep_date_col: bool = ..., + keep_date_col: bool | lib.NoDefault = ..., date_parser: Callable | lib.NoDefault = ..., date_format: str | dict[Hashable, str] | None = ..., dayfirst: bool = ..., @@ -1118,7 +1151,7 @@ def read_table( skip_blank_lines: bool = ..., parse_dates: bool | Sequence[Hashable] = ..., infer_datetime_format: bool | lib.NoDefault = ..., - keep_date_col: bool = ..., + keep_date_col: bool | lib.NoDefault = ..., date_parser: Callable | lib.NoDefault = ..., date_format: str | dict[Hashable, str] | None = ..., dayfirst: bool = ..., @@ -1175,7 +1208,7 @@ def read_table( skip_blank_lines: bool = ..., parse_dates: bool | Sequence[Hashable] = ..., infer_datetime_format: bool | lib.NoDefault = ..., - keep_date_col: bool = ..., + keep_date_col: bool | lib.NoDefault = ..., date_parser: Callable | lib.NoDefault = ..., date_format: str | dict[Hashable, str] | None = ..., dayfirst: bool = ..., @@ -1248,7 +1281,7 @@ def read_table( # Datetime Handling parse_dates: bool | Sequence[Hashable] = False, infer_datetime_format: bool | lib.NoDefault = lib.no_default, - keep_date_col: bool = False, + keep_date_col: bool | lib.NoDefault = lib.no_default, date_parser: Callable | lib.NoDefault = lib.no_default, date_format: str | dict[Hashable, str] | None = None, dayfirst: bool = False, @@ -1279,6 +1312,29 @@ def read_table( storage_options: StorageOptions | None = None, dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default, ) -> DataFrame | TextFileReader: + if keep_date_col is not lib.no_default: + # GH#55569 + warnings.warn( + "The 'keep_date_col' keyword in pd.read_table is deprecated and " + "will be removed in a future version. Explicitly remove unwanted " + "columns after parsing instead.", + FutureWarning, + stacklevel=find_stack_level(), + ) + else: + keep_date_col = False + + # error: Item "bool" of "bool | Sequence[Hashable]" has no attribute "__iter__" + if lib.is_list_like(parse_dates) and not all(is_hashable(x) for x in parse_dates): # type: ignore[union-attr] + # GH#55569 + warnings.warn( + "Support for nested sequences for 'parse_dates' in pd.read_table " + "is deprecated. Combine the desired columns with pd.to_datetime " + "after parsing instead.", + FutureWarning, + stacklevel=find_stack_level(), + ) + if infer_datetime_format is not lib.no_default: warnings.warn( "The argument 'infer_datetime_format' is deprecated and will " diff --git a/pandas/tests/io/parser/test_parse_dates.py b/pandas/tests/io/parser/test_parse_dates.py index d65961f9483d8..d8f362039ba13 100644 --- a/pandas/tests/io/parser/test_parse_dates.py +++ b/pandas/tests/io/parser/test_parse_dates.py @@ -131,13 +131,19 @@ def test_separator_date_conflict(all_parsers): [[datetime(2013, 6, 2, 13, 0, 0), 1000.215]], columns=["Date", 2] ) - df = parser.read_csv( - StringIO(data), - sep=";", - thousands="-", - parse_dates={"Date": [0, 1]}, - header=None, + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" ) + with tm.assert_produces_warning( + FutureWarning, match=depr_msg, check_stacklevel=False + ): + df = parser.read_csv( + StringIO(data), + sep=";", + thousands="-", + parse_dates={"Date": [0, 1]}, + header=None, + ) tm.assert_frame_equal(df, expected) @@ -331,13 +337,18 @@ def test_multiple_date_col(all_parsers, keep_date_col, request): ) request.applymarker(mark) + depr_msg = "The 'keep_date_col' keyword in pd.read_csv is deprecated" + kwds = { "header": None, "parse_dates": [[1, 2], [1, 3]], "keep_date_col": keep_date_col, "names": ["X0", "X1", "X2", "X3", "X4", "X5", "X6", "X7", "X8"], } - result = parser.read_csv(StringIO(data), **kwds) + with tm.assert_produces_warning( + (DeprecationWarning, FutureWarning), match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv(StringIO(data), **kwds) expected = DataFrame( [ @@ -603,7 +614,13 @@ def test_multiple_date_cols_with_header(all_parsers): KORD,19990127, 22:00:00, 21:56:00, -0.5900, 1.7100, 5.1000, 0.0000, 290.0000 KORD,19990127, 23:00:00, 22:56:00, -0.5900, 1.7100, 4.6000, 0.0000, 280.0000""" - result = parser.read_csv(StringIO(data), parse_dates={"nominal": [1, 2]}) + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" + ) + with tm.assert_produces_warning( + FutureWarning, match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv(StringIO(data), parse_dates={"nominal": [1, 2]}) expected = DataFrame( [ [ @@ -705,8 +722,14 @@ def test_multiple_date_cols_with_header(all_parsers): def test_multiple_date_col_name_collision(all_parsers, data, parse_dates, msg): parser = all_parsers + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" + ) with pytest.raises(ValueError, match=msg): - parser.read_csv(StringIO(data), parse_dates=parse_dates) + with tm.assert_produces_warning( + (FutureWarning, DeprecationWarning), match=depr_msg, check_stacklevel=False + ): + parser.read_csv(StringIO(data), parse_dates=parse_dates) def test_date_parser_int_bug(all_parsers): @@ -1101,9 +1124,15 @@ def test_multiple_date_cols_index(all_parsers, parse_dates, index_col): if not isinstance(parse_dates, dict): expected.index.name = "date_NominalTime" - result = parser.read_csv( - StringIO(data), parse_dates=parse_dates, index_col=index_col + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" ) + with tm.assert_produces_warning( + FutureWarning, match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv( + StringIO(data), parse_dates=parse_dates, index_col=index_col + ) tm.assert_frame_equal(result, expected) @@ -1187,13 +1216,19 @@ def test_multiple_date_cols_chunked(all_parsers): ) expected = expected.set_index("nominal") - with parser.read_csv( - StringIO(data), - parse_dates={"nominal": [1, 2]}, - index_col="nominal", - chunksize=2, - ) as reader: - chunks = list(reader) + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" + ) + with tm.assert_produces_warning( + FutureWarning, match=depr_msg, check_stacklevel=False + ): + with parser.read_csv( + StringIO(data), + parse_dates={"nominal": [1, 2]}, + index_col="nominal", + chunksize=2, + ) as reader: + chunks = list(reader) tm.assert_frame_equal(chunks[0], expected[:2]) tm.assert_frame_equal(chunks[1], expected[2:4]) @@ -1212,14 +1247,24 @@ def test_multiple_date_col_named_index_compat(all_parsers): KORD,19990127, 23:00:00, 22:56:00, -0.5900, 1.7100, 4.6000, 0.0000, 280.0000 """ - with_indices = parser.read_csv( - StringIO(data), parse_dates={"nominal": [1, 2]}, index_col="nominal" - ) - with_names = parser.read_csv( - StringIO(data), - index_col="nominal", - parse_dates={"nominal": ["date", "nominalTime"]}, + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" ) + with tm.assert_produces_warning( + (FutureWarning, DeprecationWarning), match=depr_msg, check_stacklevel=False + ): + with_indices = parser.read_csv( + StringIO(data), parse_dates={"nominal": [1, 2]}, index_col="nominal" + ) + + with tm.assert_produces_warning( + (FutureWarning, DeprecationWarning), match=depr_msg, check_stacklevel=False + ): + with_names = parser.read_csv( + StringIO(data), + index_col="nominal", + parse_dates={"nominal": ["date", "nominalTime"]}, + ) tm.assert_frame_equal(with_indices, with_names) @@ -1234,10 +1279,19 @@ def test_multiple_date_col_multiple_index_compat(all_parsers): KORD,19990127, 22:00:00, 21:56:00, -0.5900, 1.7100, 5.1000, 0.0000, 290.0000 KORD,19990127, 23:00:00, 22:56:00, -0.5900, 1.7100, 4.6000, 0.0000, 280.0000 """ - result = parser.read_csv( - StringIO(data), index_col=["nominal", "ID"], parse_dates={"nominal": [1, 2]} + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" ) - expected = parser.read_csv(StringIO(data), parse_dates={"nominal": [1, 2]}) + with tm.assert_produces_warning( + (FutureWarning, DeprecationWarning), match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv( + StringIO(data), index_col=["nominal", "ID"], parse_dates={"nominal": [1, 2]} + ) + with tm.assert_produces_warning( + (FutureWarning, DeprecationWarning), match=depr_msg, check_stacklevel=False + ): + expected = parser.read_csv(StringIO(data), parse_dates={"nominal": [1, 2]}) expected = expected.set_index(["nominal", "ID"]) tm.assert_frame_equal(result, expected) @@ -1866,10 +1920,21 @@ def test_missing_parse_dates_column_raises( parser = all_parsers content = StringIO("date,time,val\n2020-01-31,04:20:32,32\n") msg = f"Missing column provided to 'parse_dates': '{missing_cols}'" + + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" + ) + warn = FutureWarning + if isinstance(parse_dates, list) and all( + isinstance(x, (int, str)) for x in parse_dates + ): + warn = None + with pytest.raises(ValueError, match=msg): - parser.read_csv( - content, sep=",", names=names, usecols=usecols, parse_dates=parse_dates - ) + with tm.assert_produces_warning(warn, match=depr_msg, check_stacklevel=False): + parser.read_csv( + content, sep=",", names=names, usecols=usecols, parse_dates=parse_dates + ) @xfail_pyarrow # mismatched shape @@ -1918,11 +1983,18 @@ def test_date_parser_multiindex_columns_combine_cols(all_parsers, parse_spec, co data = """a,b,c 1,2,3 2019-12,-31,6""" - result = parser.read_csv( - StringIO(data), - parse_dates=parse_spec, - header=[0, 1], + + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" ) + with tm.assert_produces_warning( + FutureWarning, match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv( + StringIO(data), + parse_dates=parse_spec, + header=[0, 1], + ) expected = DataFrame( {col_name: Timestamp("2019-12-31").as_unit("ns"), ("c", "3"): [6]} ) @@ -1970,9 +2042,13 @@ def test_parse_dates_and_keep_original_column(all_parsers): 20150908 20150909 """ - result = parser.read_csv( - StringIO(data), parse_dates={"date": ["A"]}, keep_date_col=True - ) + depr_msg = "The 'keep_date_col' keyword in pd.read_csv is deprecated" + with tm.assert_produces_warning( + FutureWarning, match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv( + StringIO(data), parse_dates={"date": ["A"]}, keep_date_col=True + ) expected_data = [Timestamp("2015-09-08"), Timestamp("2015-09-09")] expected = DataFrame({"date": expected_data, "A": expected_data}) tm.assert_frame_equal(result, expected) @@ -2191,9 +2267,15 @@ def test_parse_dates_dict_format_two_columns(all_parsers, key, parse_dates): 31-,12-2019 31-,12-2020""" - result = parser.read_csv( - StringIO(data), date_format={key: "%d- %m-%Y"}, parse_dates=parse_dates + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" ) + with tm.assert_produces_warning( + (FutureWarning, DeprecationWarning), match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv( + StringIO(data), date_format={key: "%d- %m-%Y"}, parse_dates=parse_dates + ) expected = DataFrame( { key: [Timestamp("2019-12-31"), Timestamp("2020-12-31")], diff --git a/pandas/tests/io/parser/usecols/test_parse_dates.py b/pandas/tests/io/parser/usecols/test_parse_dates.py index 52be6f302c337..bc66189ca064e 100644 --- a/pandas/tests/io/parser/usecols/test_parse_dates.py +++ b/pandas/tests/io/parser/usecols/test_parse_dates.py @@ -34,6 +34,10 @@ def test_usecols_with_parse_dates(all_parsers, usecols): parser = all_parsers parse_dates = [[1, 2]] + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" + ) + cols = { "a": [0, 0], "c_d": [Timestamp("2014-01-01 09:00:00"), Timestamp("2014-01-02 10:00:00")], @@ -41,9 +45,19 @@ def test_usecols_with_parse_dates(all_parsers, usecols): expected = DataFrame(cols, columns=["c_d", "a"]) if parser.engine == "pyarrow": with pytest.raises(ValueError, match=_msg_pyarrow_requires_names): - parser.read_csv(StringIO(data), usecols=usecols, parse_dates=parse_dates) + with tm.assert_produces_warning( + FutureWarning, match=depr_msg, check_stacklevel=False + ): + parser.read_csv( + StringIO(data), usecols=usecols, parse_dates=parse_dates + ) return - result = parser.read_csv(StringIO(data), usecols=usecols, parse_dates=parse_dates) + with tm.assert_produces_warning( + FutureWarning, match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv( + StringIO(data), usecols=usecols, parse_dates=parse_dates + ) tm.assert_frame_equal(result, expected) @@ -127,11 +141,17 @@ def test_usecols_with_parse_dates4(all_parsers): } expected = DataFrame(cols, columns=["a_b"] + list("cdefghij")) - result = parser.read_csv( - StringIO(data), - usecols=usecols, - parse_dates=parse_dates, + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" ) + with tm.assert_produces_warning( + (FutureWarning, DeprecationWarning), match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv( + StringIO(data), + usecols=usecols, + parse_dates=parse_dates, + ) tm.assert_frame_equal(result, expected) @@ -162,7 +182,13 @@ def test_usecols_with_parse_dates_and_names(all_parsers, usecols, names, request } expected = DataFrame(cols, columns=["c_d", "a"]) - result = parser.read_csv( - StringIO(s), names=names, parse_dates=parse_dates, usecols=usecols + depr_msg = ( + "Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated" ) + with tm.assert_produces_warning( + (FutureWarning, DeprecationWarning), match=depr_msg, check_stacklevel=False + ): + result = parser.read_csv( + StringIO(s), names=names, parse_dates=parse_dates, usecols=usecols + ) tm.assert_frame_equal(result, expected)