Merge pull request #1270 from mito-ds/datetime-filter-bug

Add better handling for datetime filters

Author: marthacryan

mitosheet/mitosheet/code_chunks/step_performers/filter_code_chunk.py

@@ -6,6 +6,7 @@
 
 
 from copy import copy
+from datetime import date
 from typing import List, Optional, Tuple, Union
 
 from mitosheet.code_chunks.code_chunk import CodeChunk
@@ -25,6 +26,7 @@
     get_column_header_as_transpiled_code, get_list_as_string_without_internal_quotes)
 from mitosheet.types import (ColumnHeader, ColumnID, ColumnIDWithFilterGroup,
                              Filter, FilterGroup, OperatorType)
+import pandas as pd
 
 # Dict used when a filter condition is only used by one filter
 FILTER_FORMAT_STRING_DICT = {
@@ -180,7 +182,7 @@
 OPERATOR_SIGNS = {"Or": "|", "And": "&"}
 
 def get_single_filter_string(
-    df_name: str, column_header: ColumnHeader, filter_: Filter
+    df_name: str, column_header: ColumnHeader, column_dtype: str, filter_: Filter
 ) -> str:
     """
     Transpiles a specific filter to a fitler string, to be used
@@ -189,6 +191,14 @@ def get_single_filter_string(
     condition = filter_["condition"]
     value = filter_["value"]
 
+    # When users are mid-way through typing a date in the UI, it might not be valid yet.
+    # To prevent throwing errors, we set it to the minimum date
+    if is_datetime_dtype(column_dtype):
+        try: 
+            pd.to_datetime(value)
+        except:
+            value = pd.Timestamp.min + pd.Timedelta(seconds=1)
+
     transpiled_column_header = get_column_header_as_transpiled_code(column_header)
     value = get_column_header_as_transpiled_code(value)
 
@@ -289,7 +299,7 @@ def create_filter_string_for_condition(
     elif len(filters_with_condition) == 1:
         # Use the single filter condition
         return get_single_filter_string(
-            df_name, column_header, filters_with_condition[0]
+            df_name, column_header, column_dtype, filters_with_condition[0]
         )
     elif len(filters_with_condition) > 1:
         # Use the multiple filter condition

mitosheet/mitosheet/code_chunks/step_performers/pivot_code_chunk.py

@@ -155,7 +155,11 @@ def get_code(self) -> Tuple[List[str], List[str]]:
             for column_id, value in self.values_column_ids_map.items()
         }
         pivot_filters: List[ColumnHeaderWithFilter] = [
-            {'column_header': self.prev_state.column_ids.get_column_header_by_id(self.sheet_index, pf['column_id']), 'filter': pf['filter']}
+            {
+                'column_header': self.prev_state.column_ids.get_column_header_by_id(self.sheet_index, pf['column_id']), 
+                'column_dtype': str(self.prev_state.dfs[self.sheet_index][pf['column_id']].dtype),
+                'filter': pf['filter']
+            }
             for pf in self.pivot_filters_ids
         ]
 
@@ -168,7 +172,7 @@ def get_code(self) -> Tuple[List[str], List[str]]:
         # First, filter down to the rows of the original dataframe that we need
         if len(pivot_filters) > 0:
             filter_strings = [
-                get_single_filter_string(self.old_df_name, pf['column_header'], pf['filter'])
+                get_single_filter_string(self.old_df_name, pf['column_header'], pf['column_dtype'], pf['filter'])
                 for pf in pivot_filters
             ]
             full_filter_string = combine_filter_strings('And', filter_strings)

mitosheet/mitosheet/tests/saved_analyses/test_save_utils.py

@@ -14,7 +14,7 @@
 import pytest
 from mitosheet.saved_analyses import SAVED_ANALYSIS_FOLDER, write_save_analysis_file
 from mitosheet.saved_analyses.save_utils import read_and_upgrade_analysis
-from mitosheet.step_performers.filter import FC_NUMBER_EXACTLY
+from mitosheet.types import FC_NUMBER_EXACTLY
 from mitosheet.tests.test_utils import (create_mito_wrapper_with_data,
                                         create_mito_wrapper)
 

mitosheet/mitosheet/tests/step_performers/test_filter.py

@@ -322,6 +322,16 @@
             data={"A": pd.to_datetime(pd.Series(data=["12-2-2020", "12-3-2020"]))}
         ),
     ),
+    # Test for when the year is not fully inputted yet so the date is invalid. 
+    # We don't throw an error for this so that the user can continue inputting the date.
+    (
+        pd.DataFrame(
+            data={"A": pd.to_datetime(pd.Series(data=["12-2-2020", "12-3-2020"]))}
+        ),
+        FC_DATETIME_LESS_THAN_OR_EQUAL,
+        "12-2-0020",
+        pd.DataFrame(data={"A": pd.to_datetime(pd.Series(data=[], dtype='float64'))}),
+    ),
     (
         pd.DataFrame(
             data={"A": pd.to_datetime(pd.Series(data=["12-2-2020", "12-3-2020"]))}

mitosheet/mitosheet/tests/step_performers/test_pivot.py

@@ -15,7 +15,7 @@
 from pandas.testing import assert_frame_equal
 
 from mitosheet.saved_analyses import read_and_upgrade_analysis
-from mitosheet.step_performers.filter import (FC_BOOLEAN_IS_TRUE,
+from mitosheet.types import (FC_BOOLEAN_IS_TRUE,
                                               FC_DATETIME_EXACTLY,
                                               FC_NUMBER_EXACTLY,
                                               FC_NUMBER_GREATER,
@@ -34,7 +34,6 @@
 from mitosheet.tests.decorators import pandas_post_1_only, pandas_pre_1_only
 from mitosheet.tests.test_utils import (create_mito_wrapper,
                                         get_dataframe_generation_code)
-from mitosheet.types import ColumnHeaderWithPivotTransform
 
 
 def test_simple_pivot():

mitosheet/mitosheet/tests/step_performers/test_set_dataframe_format.py

@@ -11,9 +11,8 @@
 import pandas as pd
 import pytest
 from mitosheet.state import NUMBER_FORMAT_PLAIN_TEXT, NUMBER_FORMAT_CURRENCY, get_default_dataframe_format
-from mitosheet.step_performers.filter import FC_NUMBER_GREATER
 from mitosheet.tests.test_utils import create_mito_wrapper
-from mitosheet.types import ConditionalFormat, DataframeFormat
+from mitosheet.types import ConditionalFormat, DataframeFormat, FC_NUMBER_GREATER
 
 
 

mitosheet/mitosheet/tests/updates/test_undo.py

@@ -8,7 +8,7 @@
 """
 import pandas as pd
 
-from mitosheet.step_performers.filter import FC_NUMBER_EXACTLY
+from mitosheet.types import FC_NUMBER_EXACTLY
 from mitosheet.tests.test_utils import create_mito_wrapper_with_data, create_mito_wrapper
 
 

mitosheet/mitosheet/types.py

@@ -204,6 +204,7 @@ class ColumnIDWithFilterGroup(TypedDict):
 
     class ColumnHeaderWithFilter(TypedDict):
         column_header: ColumnHeader
+        column_dtype: str
         filter: Filter
 
     class ColumnIDWithPivotTransform(TypedDict):