diff --git a/mitosheet/mitosheet/code_chunks/postprocessing/set_dataframe_format_code_chunk.py b/mitosheet/mitosheet/code_chunks/postprocessing/set_dataframe_format_code_chunk.py index ac009bf8a..8161b57aa 100644 --- a/mitosheet/mitosheet/code_chunks/postprocessing/set_dataframe_format_code_chunk.py +++ b/mitosheet/mitosheet/code_chunks/postprocessing/set_dataframe_format_code_chunk.py @@ -180,6 +180,7 @@ def get_conditional_format_code_list(state: State, sheet_index: int) -> Tuple[Op """Returns all the code to set the conditional formats""" df_name = state.df_names[sheet_index] df = state.dfs[sheet_index] + conditional_formats = state.df_formats[sheet_index]['conditional_formats'] # We get the conditional formatting results, and we filter out any columns that are diff --git a/mitosheet/mitosheet/mito_backend.py b/mitosheet/mitosheet/mito_backend.py index a1e980060..23e1ae492 100644 --- a/mitosheet/mitosheet/mito_backend.py +++ b/mitosheet/mitosheet/mito_backend.py @@ -29,7 +29,7 @@ from mitosheet.steps_manager import StepsManager from mitosheet.telemetry.telemetry_utils import (log, log_event_processed, telemetry_turned_on) -from mitosheet.types import CodeOptions, MitoTheme, ParamMetadata +from mitosheet.types import CodeOptions, ColumnDefinintion, ColumnDefinitions, ConditionalFormat, MitoTheme, ParamMetadata from mitosheet.updates.replay_analysis import REPLAY_ANALYSIS_UPDATE from mitosheet.user.create import try_create_user_json_file from mitosheet.user.db import USER_JSON_PATH, get_user_field @@ -57,6 +57,7 @@ def __init__( user_defined_importers: Optional[List[Callable]]=None, user_defined_editors: Optional[List[Callable]]=None, code_options: Optional[CodeOptions]=None, + column_definitions: Optional[List[ColumnDefinitions]]=None, theme: Optional[MitoTheme]=None, ): """ @@ -92,7 +93,7 @@ def __init__( if not os.path.exists(import_folder): raise ValueError(f"Import folder {import_folder} does not exist. Please change the file path or create the folder.") - + # Set up the state container to hold private widget state self.steps_manager = StepsManager( args, @@ -104,6 +105,7 @@ def __init__( user_defined_importers=all_custom_importers, user_defined_editors=user_defined_editors, code_options=code_options, + column_definitions=column_definitions, theme=theme ) @@ -314,6 +316,7 @@ def get_mito_backend( user_defined_functions: Optional[List[Callable]]=None, user_defined_importers: Optional[List[Callable]]=None, user_defined_editors: Optional[List[Callable]]=None, + column_definitions: Optional[List[ColumnDefinitions]]=None, ) -> MitoBackend: # We pass in the dataframes directly to the widget @@ -322,7 +325,8 @@ def get_mito_backend( analysis_to_replay=analysis_to_replay, user_defined_functions=user_defined_functions, user_defined_importers=user_defined_importers, - user_defined_editors=user_defined_editors + user_defined_editors=user_defined_editors, + column_definitions=column_definitions ) return mito_backend diff --git a/mitosheet/mitosheet/pro/conditional_formatting_utils.py b/mitosheet/mitosheet/pro/conditional_formatting_utils.py index ff040f186..b10f74663 100644 --- a/mitosheet/mitosheet/pro/conditional_formatting_utils.py +++ b/mitosheet/mitosheet/pro/conditional_formatting_utils.py @@ -52,6 +52,7 @@ def get_conditonal_formatting_result( # are sent to the frontend json_index = json.dumps(index, cls=NpEncoder) formatted_result[column_id][json_index] = {'backgroundColor': backgroundColor, 'color': color} + except Exception as e: if format_uuid not in invalid_conditional_formats: invalid_conditional_formats[format_uuid] = [] diff --git a/mitosheet/mitosheet/step_performers/filter.py b/mitosheet/mitosheet/step_performers/filter.py index 1ebf67ac0..b97dd624f 100644 --- a/mitosheet/mitosheet/step_performers/filter.py +++ b/mitosheet/mitosheet/step_performers/filter.py @@ -41,6 +41,7 @@ FC_NUMBER_HIGHEST, ] + class FilterStepPerformer(StepPerformer): """ Allows you to filter a column based on some conditions and some values. @@ -260,5 +261,5 @@ def check_filters_contain_condition_that_needs_full_df(filters: List[Union[Filte filter_group: FilterGroup = filter_or_group #type: ignore if check_filters_contain_condition_that_needs_full_df(filter_group["filters"]): return True - + return False diff --git a/mitosheet/mitosheet/steps_manager.py b/mitosheet/mitosheet/steps_manager.py index a872deeb9..1b46e7a55 100644 --- a/mitosheet/mitosheet/steps_manager.py +++ b/mitosheet/mitosheet/steps_manager.py @@ -35,10 +35,10 @@ SnowflakeImportStepPerformer from mitosheet.transpiler.transpile import transpile from mitosheet.transpiler.transpile_utils import get_default_code_options -from mitosheet.types import CodeOptions, MitoTheme, ParamMetadata +from mitosheet.types import CodeOptions, ColumnDefinintion, ColumnDefinitions, MitoTheme, ParamMetadata from mitosheet.updates import UPDATES from mitosheet.user.utils import is_enterprise, is_running_test -from mitosheet.utils import NpEncoder, dfs_to_array_for_json, get_new_id, is_default_df_names +from mitosheet.utils import NpEncoder, dfs_to_array_for_json, get_default_df_formats, get_new_id, is_default_df_names from mitosheet.step_performers.utils.user_defined_function_utils import get_user_defined_importers_for_frontend, get_user_defined_editors_for_frontend from mitosheet.step_performers.utils.user_defined_function_utils import validate_and_wrap_sheet_functions, validate_user_defined_editors @@ -187,6 +187,7 @@ def __init__( user_defined_importers: Optional[List[Callable]]=None, user_defined_editors: Optional[List[Callable]]=None, code_options: Optional[CodeOptions]=None, + column_definitions: Optional[List[ColumnDefinitions]]=None, theme: Optional[MitoTheme]=None, ): """ @@ -221,11 +222,11 @@ def __init__( # saving any data that we need to transpilate it later this self.preprocess_execution_data = {} df_names = None - for preprocess_step_performers in PREPROCESS_STEP_PERFORMERS: - args, df_names, execution_data = preprocess_step_performers.execute(args) + for preprocess_step_performer in PREPROCESS_STEP_PERFORMERS: + args, df_names, execution_data = preprocess_step_performer.execute(args) self.preprocess_execution_data[ - preprocess_step_performers.preprocess_step_type() - ] = execution_data + preprocess_step_performer.preprocess_step_type() + ] = execution_data # We set the original_args_raw_strings. If we later have an args update, then these # are overwritten by the args update (and are actually correct). But since we don't @@ -255,6 +256,8 @@ def __init__( # The version of the public interface used by this analysis self.public_interface_version = 3 + df_formats = get_default_df_formats(column_definitions, list(args)) + # Then we initialize the analysis with just a simple initialize step self.steps_including_skipped: List[Step] = [ Step( @@ -265,7 +268,8 @@ def __init__( df_names=df_names, user_defined_functions=self.user_defined_functions, user_defined_importers=self.user_defined_importers, - user_defined_editors=self.user_defined_editors + user_defined_editors=self.user_defined_editors, + df_formats=df_formats ), {} ) diff --git a/mitosheet/mitosheet/streamlit/v1/apps/column_definitions_app.py b/mitosheet/mitosheet/streamlit/v1/apps/column_definitions_app.py new file mode 100644 index 000000000..b4fbb3c39 --- /dev/null +++ b/mitosheet/mitosheet/streamlit/v1/apps/column_definitions_app.py @@ -0,0 +1,37 @@ +import pandas as pd +import streamlit as st +from mitosheet.streamlit.v1 import spreadsheet + +st.set_page_config(layout="wide") +st.title('Tesla Stock Volume Analysis') + +df = pd.DataFrame({ + 'A': [1, 2, 3, 4, 5, 6, 7, 8, 9], + 'B': [1, 2, 3, 4, 5, 6, 7, 8, 9] +}) +new_dfs, code = spreadsheet( + df, + column_definitions=[ + [ + { + 'columns': ['A', 'B'], + 'conditional_formats': [{ + 'filters': [{'condition': 'greater_than_or_equal', 'value': 5}], + 'font_color': '#c30010', + 'background_color': '#ffcbd1' + }] + }, + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'less', 'value': 2}], + 'font_color': '#f30010', + 'background_color': '#ddcbd1' + }] + } + ], + ] +) + +st.write(new_dfs) +st.code(code) \ No newline at end of file diff --git a/mitosheet/mitosheet/streamlit/v1/apps/simple.py b/mitosheet/mitosheet/streamlit/v1/apps/simple_dataframe_app.py similarity index 84% rename from mitosheet/mitosheet/streamlit/v1/apps/simple.py rename to mitosheet/mitosheet/streamlit/v1/apps/simple_dataframe_app.py index 771c73ddf..7e9c629a1 100644 --- a/mitosheet/mitosheet/streamlit/v1/apps/simple.py +++ b/mitosheet/mitosheet/streamlit/v1/apps/simple_dataframe_app.py @@ -9,7 +9,7 @@ 'a': [1, 2, 3], 'b': [4, 5, 6] }) -new_dfs, code = spreadsheet(df, height='1000px') +new_dfs, code = spreadsheet(df) st.write(new_dfs) st.code(code) \ No newline at end of file diff --git a/mitosheet/mitosheet/streamlit/v1/spreadsheet.py b/mitosheet/mitosheet/streamlit/v1/spreadsheet.py index f8d49ecae..98a1fbb45 100644 --- a/mitosheet/mitosheet/streamlit/v1/spreadsheet.py +++ b/mitosheet/mitosheet/streamlit/v1/spreadsheet.py @@ -13,7 +13,7 @@ from mitosheet.mito_backend import MitoBackend from mitosheet.selection_utils import get_selected_element -from mitosheet.types import CodeOptions, ParamMetadata, ParamType +from mitosheet.types import CodeOptions, ColumnDefinitions, ConditionalFormat, ParamMetadata, ParamType from mitosheet.utils import get_new_id CURRENT_MITO_ANALYSIS_VERSION = 1 @@ -269,6 +269,7 @@ def _get_mito_backend( _editors: Optional[List[Callable]]=None, _sheet_functions: Optional[List[Callable]]=None, _code_options: Optional[CodeOptions]=None, + _column_definitions: Optional[List[ColumnDefinitions]]=None, import_folder: Optional[str]=None, df_names: Optional[List[str]]=None, session_id: Optional[str]=None, @@ -280,6 +281,7 @@ def _get_mito_backend( import_folder=import_folder, user_defined_importers=_importers, user_defined_functions=_sheet_functions, user_defined_editors=_editors, code_options=_code_options, + column_definitions = _column_definitions, ) # Make a send function that stores the responses in a list @@ -320,6 +322,7 @@ def spreadsheet( # type: ignore df_names: Optional[List[str]]=None, import_folder: Optional[str]=None, code_options: Optional[CodeOptions]=None, + column_definitions: Optional[List[ColumnDefinitions]]=None, return_type: str='default', height: Optional[str]=None, key=None @@ -366,6 +369,7 @@ def spreadsheet( # type: ignore _importers=importers, _editors=editors, _code_options=code_options, + _column_definitions=column_definitions, import_folder=import_folder, session_id=session_id, df_names=df_names, diff --git a/mitosheet/mitosheet/tests/api/test_get_code_snippets.py b/mitosheet/mitosheet/tests/api/test_get_code_snippets.py index bc7ad1f0e..971c5070a 100644 --- a/mitosheet/mitosheet/tests/api/test_get_code_snippets.py +++ b/mitosheet/mitosheet/tests/api/test_get_code_snippets.py @@ -1,10 +1,11 @@ import json + +from pytest_httpserver import HTTPServer from mitosheet.enterprise.mito_config import MITO_CONFIG_CODE_SNIPPETS_URL, MITO_CONFIG_CODE_SNIPPETS_VERSION, MITO_CONFIG_VERSION from mitosheet.tests.test_mito_config import delete_all_mito_config_environment_variables from mitosheet.tests.test_utils import create_mito_wrapper from mitosheet.api.get_code_snippets import get_code_snippets, DEFAULT_CODE_SNIPPETS import os -from pytest_httpserver import HTTPServer TEST_CODE_SNIPPETS = [ diff --git a/mitosheet/mitosheet/tests/streamlit/test_runnable_analysis.py b/mitosheet/mitosheet/tests/streamlit/test_runnable_analysis.py index f6d891b90..11a8f56dd 100644 --- a/mitosheet/mitosheet/tests/streamlit/test_runnable_analysis.py +++ b/mitosheet/mitosheet/tests/streamlit/test_runnable_analysis.py @@ -85,7 +85,7 @@ def function(file_name_import_csv_0, file_name_import_excel_0, file_name_export_ } ] - analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) + analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) result = analysis.run() assert result is not None pd.testing.assert_frame_equal(result[0], expected_df) @@ -154,7 +154,7 @@ def function(file_name_import_csv_0, file_name_import_excel_0, file_name_export_ new_export_file_0 = str(tmp_path / 'new_export.csv') new_export_file_1 = str(tmp_path / 'new_export.xlsx') - analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) + analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) # Test that get_param_metadata works assert analysis.get_param_metadata() == param_metadata @@ -192,7 +192,7 @@ def function_ctqm(import_dataframe_0): } ] - analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) + analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) assert analysis.get_param_metadata() == param_metadata assert analysis.get_param_metadata('import') == param_metadata @@ -273,7 +273,7 @@ def function(import_dataframe_0, file_name_import_csv_0, file_name_import_excel_ new_export_file_0 = str(tmp_path / 'new_export.csv') new_export_file_1 = str(tmp_path / 'new_export.xlsx') - analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) + analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) # Test that get_param_metadata works assert analysis.get_param_metadata() == param_metadata @@ -383,7 +383,7 @@ def function(import_dataframe_0): }, ] - analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) + analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) with pytest.raises(TypeError): analysis.run() @@ -406,14 +406,14 @@ def function(import_dataframe_0): } ] - analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) + analysis = RunnableAnalysis('', None, fully_parameterized_function, param_metadata) with pytest.raises(TypeError): analysis.run(testing=1) @requires_streamlit def test_to_and_from_json(): - analysis = RunnableAnalysis('', None, simple_fn, simple_param_metadata) + analysis = RunnableAnalysis('', None, simple_fn, simple_param_metadata) # Test that the to_json function json = analysis.to_json() assert json is not None @@ -445,7 +445,7 @@ def function(vari\abl"e_name{}): 'name': 'vari\ abl"e_name{}' }, ] - analysis = RunnableAnalysis('', None, special_characters_fn, special_characters_metadata) + analysis = RunnableAnalysis('', None, special_characters_fn, special_characters_metadata) # Test that the to_json function json = analysis.to_json() assert json is not None @@ -467,7 +467,7 @@ def function(): return df1 """ - analysis = RunnableAnalysis('', None, fully_parameterized_code, []) + analysis = RunnableAnalysis('', None, fully_parameterized_code, []) result = analysis.run() pd.testing.assert_frame_equal(result, pd.DataFrame({'A': [1, 2, 3], 'B': [2, 3, 4]})) diff --git a/mitosheet/mitosheet/tests/streamlit/test_streamlit.py b/mitosheet/mitosheet/tests/streamlit/test_streamlit.py index 3cc9f836d..df2c5f141 100644 --- a/mitosheet/mitosheet/tests/streamlit/test_streamlit.py +++ b/mitosheet/mitosheet/tests/streamlit/test_streamlit.py @@ -62,7 +62,7 @@ 'return type of analysis', [df1], {'return_type': 'analysis'}, ( - RunnableAnalysis('', None, '', []) + RunnableAnalysis('', None, '', [], 0) ) ) ] @@ -109,3 +109,32 @@ def test_return_type_function_invalid_code_options(): spreadsheet(df1, code_options={'as_function': False, 'call_function': False, 'function_name': 'test', 'function_params': {}}, return_type='function') with pytest.raises(ValueError): spreadsheet(df1, code_options={'as_function': True, 'call_function': True, 'function_name': 'test', 'function_params': {}}, return_type='function') + +@requires_streamlit +def test_spreadsheet_with_column_definitions(): + f = spreadsheet( + df1, + column_definitions=[ + [ + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'greater_than_or_equal', 'value': 5}], + 'font_color': '#c30010', + 'background_color': '#ffcbd1' + }] + }, + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'less', 'value': 2}], + 'font_color': '#f30010', + 'background_color': '#ddcbd1' + }] + } + ] + ], + code_options={'as_function': True, 'call_function': False, 'function_name': 'test', 'function_params': {}}, + return_type='function' + ) + assert callable(f) \ No newline at end of file diff --git a/mitosheet/mitosheet/tests/test_column_definitions.py b/mitosheet/mitosheet/tests/test_column_definitions.py new file mode 100644 index 000000000..efe4738cb --- /dev/null +++ b/mitosheet/mitosheet/tests/test_column_definitions.py @@ -0,0 +1,240 @@ +import pytest +from mitosheet.mito_backend import MitoBackend, get_mito_backend +import pandas as pd + +df = pd.DataFrame({ + 'A': [1, 2, 3, 4, 5, 6, 7, 8, 9], + 'B': [1, 2, 3, 4, 5, 6, 7, 8, 9] +}) + +def test_create_backend_with_column_definitions_works(): + + + column_definitions= [ + [ + { + 'columns': ['A', 'B'], + 'conditional_formats': [{ + 'filters': [{'condition': 'greater', 'value': 5}], + 'font_color': '#c30010', + 'background_color': '#ffcbd1' + }] + }, + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'less_than_or_equal', 'value': 0}], + 'font_color': '#FFFFFF', + 'background_color': '#000000' + }] + }, + ] + ] + + mito_backend = get_mito_backend(df, column_definitions=column_definitions) + + df_formats = mito_backend.steps_manager.curr_step.df_formats + + # Still generates valid, empty df_formats objects + assert df_formats[0]['columns'] == {} + assert df_formats[0]['headers'] == {} + assert df_formats[0]['rows']['even'] == {} + assert df_formats[0]['rows']['odd'] == {} + assert df_formats[0]['border'] == {} + + # But now has conditional formats + assert len(df_formats[0]['conditional_formats']) == 2 + assert df_formats[0]['conditional_formats'][0]['columnIDs'] == ['A', 'B'] + assert df_formats[0]['conditional_formats'][0]['filters'] == [{'condition': 'greater', 'value': 5}] + assert df_formats[0]['conditional_formats'][0]['color'] == '#c30010' + assert df_formats[0]['conditional_formats'][0]['backgroundColor'] == '#ffcbd1' + assert df_formats[0]['conditional_formats'][0]['invalidFilterColumnIDs'] == [] + + assert df_formats[0]['conditional_formats'][1]['columnIDs'] == ['A'] + assert df_formats[0]['conditional_formats'][1]['filters'] == [{'condition': 'less_than_or_equal', 'value': 0}] + assert df_formats[0]['conditional_formats'][1]['color'] == '#FFFFFF' + assert df_formats[0]['conditional_formats'][1]['backgroundColor'] == '#000000' + assert df_formats[0]['conditional_formats'][1]['invalidFilterColumnIDs'] == [] + + # They should have different format_uuids + assert df_formats[0]['conditional_formats'][0]['format_uuid'] != df_formats[0]['conditional_formats'][1]['format_uuid'] + + +def test_create_backend_with_column_definitions_works_multiple_sheets(): + + column_definitions= [ + [ + { + 'columns': ['A', 'B'], + 'conditional_formats': [{ + 'filters': [{'condition': 'greater', 'value': 5}], + 'font_color': '#c30010', + 'background_color': '#ffcbd1' + }] + } + ], + [ + { + 'columns': ['A', 'B'], + 'conditional_formats': [{ + 'filters': [{'condition': 'greater_than_or_equal', 'value': 10}], + 'font_color': '#000000', + 'background_color': '#FFFFFF' + }] + }, + ] + ] + + dfs = [df, df] + mito_backend = get_mito_backend(*dfs, column_definitions=column_definitions) + + df_formats = mito_backend.steps_manager.curr_step.df_formats + + # Still generates valid, empty df_formats objects + assert df_formats[0]['columns'] == {} + assert df_formats[0]['headers'] == {} + assert df_formats[0]['rows']['even'] == {} + assert df_formats[0]['rows']['odd'] == {} + assert df_formats[0]['border'] == {} + + # But now has conditional formats + assert len(df_formats[0]['conditional_formats']) == 1 + assert df_formats[0]['conditional_formats'][0]['columnIDs'] == ['A', 'B'] + assert df_formats[0]['conditional_formats'][0]['filters'] == [{'condition': 'greater', 'value': 5}] + assert df_formats[0]['conditional_formats'][0]['color'] == '#c30010' + assert df_formats[0]['conditional_formats'][0]['backgroundColor'] == '#ffcbd1' + assert df_formats[0]['conditional_formats'][0]['invalidFilterColumnIDs'] == [] + + assert len(df_formats[1]['conditional_formats']) == 1 + assert df_formats[1]['conditional_formats'][0]['columnIDs'] == ['A', 'B'] + assert df_formats[1]['conditional_formats'][0]['filters'] == [{'condition': 'greater_than_or_equal', 'value': 10}] + assert df_formats[1]['conditional_formats'][0]['color'] == '#000000' + assert df_formats[1]['conditional_formats'][0]['backgroundColor'] == '#FFFFFF' + assert df_formats[1]['conditional_formats'][0]['invalidFilterColumnIDs'] == [] + + # They should have different format_uuids + assert df_formats[0]['conditional_formats'][0]['format_uuid'] != df_formats[1]['conditional_formats'][0]['format_uuid'] + + +INVALID_COLUMN_DEFINITIONS = [ + ( + [ + [ + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'less_than_or_equal', 'value': 0}], + 'font_color': 'ABC', + 'background_color': '#000000' + }] + }, + ] + ], "set in column_definititon is not a valid hex color" + ), + ( + [ + [ + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'less_than_or_equal', 'value': 0}], + }] + }, + ] + ], + "column_definititon has invalid conditional_format rules. It must set the font_color, background_color, or both." + ), + ( + [ + [ + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'less_than_or_equal', 'value': 0}], + 'font_color': '#000000', + 'background_color': '#000000' + }] + }, + ], + [ + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'less_than_or_equal', 'value': 0}], + 'font_color': '#000000', + 'background_color': '#000000' + }] + }, + ], + ], + "dataframes are provided." + ), + ( + [ + [ + { + 'columns': ['D'], + 'conditional_formats': [{ + 'filters': [{'condition': 'less_than_or_equal', 'value': 0}], + 'font_color': '#000000', + 'background_color': '#000000' + }] + }, + ], + ], + "don't exist in the dataframe." + ), + ( + [ + [ + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': 'INVALID_FILTER', 'value': 0}], + 'font_color': '#000000', + 'background_color': '#000000' + }] + }, + ], + ], + "The condition INVALID_FILTER is not a valid filter condition." + ), +] +@pytest.mark.parametrize("column_definitions,error", INVALID_COLUMN_DEFINITIONS) +def test_invalid_column_definitions(column_definitions, error): + + with pytest.raises(ValueError) as e_info: + mito_backend = get_mito_backend(df, column_definitions=column_definitions) + + assert error in str(e_info) + + +def test_create_backend_with_column_definitions_does_not_error_with_mismatch_condition_and_column_type(): + column_definitions= [ + [ + { + 'columns': ['A'], + 'conditional_formats': [{ + 'filters': [{'condition': "string_does_not_contain", 'value': 0}], + 'font_color': '#c30010', + 'background_color': '#ffcbd1' + }] + }, + ], + ] + + mito_backend = get_mito_backend(df, column_definitions=column_definitions) + df_formats = mito_backend.steps_manager.curr_step.df_formats + assert len(df_formats[0]['conditional_formats']) == 1 + assert df_formats[0]['conditional_formats'][0]['columnIDs'] == ['A'] + assert df_formats[0]['conditional_formats'][0]['filters'] == [{'condition': "string_does_not_contain", 'value': 0}] + assert df_formats[0]['conditional_formats'][0]['color'] == '#c30010' + assert df_formats[0]['conditional_formats'][0]['backgroundColor'] == '#ffcbd1' + assert df_formats[0]['conditional_formats'][0]['invalidFilterColumnIDs'] == [] + + + + + + + diff --git a/mitosheet/mitosheet/tests/test_mito_backend.py b/mitosheet/mitosheet/tests/test_mito_backend.py index 60f71e908..093158650 100644 --- a/mitosheet/mitosheet/tests/test_mito_backend.py +++ b/mitosheet/mitosheet/tests/test_mito_backend.py @@ -47,6 +47,7 @@ def test_example_creation_blank(): ] @pytest.mark.parametrize("df", VALID_DATAFRAMES) def test_df_creates_valid_df(df): + print(df) print(type(df[0])) mito = create_mito_wrapper(df[0]) assert mito.mito_backend is not None @@ -188,4 +189,6 @@ def test_create_backend_with_code_options_works(): code_options = get_default_code_options('tmp') code_options['call_function'] = False mito_backend = MitoBackend(code_options=code_options) - assert mito_backend.steps_manager.code_options == code_options \ No newline at end of file + assert mito_backend.steps_manager.code_options == code_options + + \ No newline at end of file diff --git a/mitosheet/mitosheet/transpiler/transpile.py b/mitosheet/mitosheet/transpiler/transpile.py index 2f4420853..cb720c3f8 100644 --- a/mitosheet/mitosheet/transpiler/transpile.py +++ b/mitosheet/mitosheet/transpiler/transpile.py @@ -13,8 +13,8 @@ from mitosheet.code_chunks.code_chunk import CodeChunk from mitosheet.code_chunks.code_chunk_utils import get_code_chunks from mitosheet.code_chunks.postprocessing import POSTPROCESSING_CODE_CHUNKS - from mitosheet.preprocessing import PREPROCESS_STEP_PERFORMERS + from mitosheet.transpiler.transpile_utils import get_script_as_function, get_imports_for_custom_python_code from mitosheet.types import StepsManagerType, CodeOptions diff --git a/mitosheet/mitosheet/types.py b/mitosheet/mitosheet/types.py index 8ee6f1d32..b7acb9642 100644 --- a/mitosheet/mitosheet/types.py +++ b/mitosheet/mitosheet/types.py @@ -373,6 +373,18 @@ class CodeOptions(TypedDict): # The params below become optional. Typing them is hard, so use care when accessing them import_custom_python_code: bool + class ColumnDefinitionConditionalFormats(TypedDict): + filters: List[Filter] + background_color: str + font_color: str + + class ColumnDefinintion(TypedDict): + columns: List[ColumnHeader] + conditional_formats: List[ColumnDefinitionConditionalFormats] + + # TODO: This is a confusing name. Think of a better one. + ColumnDefinitions = List[ColumnDefinintion] + UserDefinedFunctionParamType = Literal['any', 'str', 'int', 'float', 'bool', 'DataFrame', 'ColumnHeader'] class MitoTheme(TypedDict): @@ -439,6 +451,9 @@ class ExecuteThroughTranspileNewDataframeParams(TypedDict): MitoTheme = Any # type: ignore MitoFrontendSelection = Any # type: ignore MitoFrontendIndexAndSelections = Any # type: ignore + ColumnDefinitionConditionalFormats = Any # type: ignore + ColumnDefinintion = Any # type: ignore + ColumnDefinitions = Any # type: ignore ParamName = str # type: ignore ParamType = str # type: ignore diff --git a/mitosheet/mitosheet/utils.py b/mitosheet/mitosheet/utils.py index a728bc389..3f9624a85 100644 --- a/mitosheet/mitosheet/utils.py +++ b/mitosheet/mitosheet/utils.py @@ -9,6 +9,7 @@ import json import pprint from random import randint +import random import re import uuid from typing import Any, Dict, List, Optional, Set, Tuple @@ -20,12 +21,21 @@ from mitosheet.column_headers import ColumnIDMap, get_column_header_display from mitosheet.is_type_utils import get_float_dt_td_columns, is_int_dtype -from mitosheet.types import (ColumnHeader, ColumnID, DataframeFormat, FrontendFormulaAndLocation, StateType) +from mitosheet.types import (FC_BOOLEAN_IS_FALSE, FC_BOOLEAN_IS_TRUE, FC_DATETIME_EXACTLY, FC_DATETIME_GREATER, FC_DATETIME_GREATER_THAN_OR_EQUAL, FC_DATETIME_LESS, + FC_DATETIME_LESS_THAN_OR_EQUAL, FC_DATETIME_NOT_EXACTLY, FC_EMPTY, + FC_LEAST_FREQUENT, FC_MOST_FREQUENT, FC_NOT_EMPTY, FC_NUMBER_EXACTLY, + FC_NUMBER_GREATER, FC_NUMBER_GREATER_THAN_OR_EQUAL, FC_NUMBER_HIGHEST, + FC_NUMBER_LESS, FC_NUMBER_LESS_THAN_OR_EQUAL, FC_NUMBER_LOWEST, + FC_NUMBER_NOT_EXACTLY, FC_STRING_CONTAINS, FC_STRING_DOES_NOT_CONTAIN, + FC_STRING_ENDS_WITH, FC_STRING_EXACTLY, FC_STRING_NOT_EXACTLY, + FC_STRING_STARTS_WITH, FC_STRING_CONTAINS_CASE_INSENSITIVE, + ColumnDefinitionConditionalFormats, ColumnDefinitions, + ColumnHeader, ColumnID, ConditionalFormat, DataframeFormat, + FrontendFormulaAndLocation, StateType) from mitosheet.excel_utils import get_df_name_as_valid_sheet_name from mitosheet.public.v3.formatting import add_formatting_to_excel_sheet - # We only send the first 1500 rows of a dataframe; note that this # must match this variable defined on the front-end MAX_ROWS = 1_500 @@ -267,7 +277,104 @@ def write_to_excel( number_formats=get_number_formats_objects_to_export_to_excel(df, format.get('columns')) ) +def is_valid_hex_color(color: str) -> bool: + + if not color.startswith('#'): + return False + + match = re.search(r'^#(?:[0-9a-fA-F]{3}){1,2}$', color) + return match is not None + +def is_valid_filter_condition(filter_condition: str) -> bool: + return filter_condition in [ + FC_BOOLEAN_IS_FALSE, FC_BOOLEAN_IS_TRUE, FC_DATETIME_EXACTLY, + FC_DATETIME_GREATER, FC_DATETIME_GREATER_THAN_OR_EQUAL, FC_DATETIME_LESS, + FC_DATETIME_LESS_THAN_OR_EQUAL, FC_DATETIME_NOT_EXACTLY, FC_EMPTY, + FC_LEAST_FREQUENT, FC_MOST_FREQUENT, FC_NOT_EMPTY, FC_NUMBER_EXACTLY, + FC_NUMBER_GREATER, FC_NUMBER_GREATER_THAN_OR_EQUAL, FC_NUMBER_HIGHEST, + FC_NUMBER_LESS, FC_NUMBER_LESS_THAN_OR_EQUAL, FC_NUMBER_LOWEST, + FC_NUMBER_NOT_EXACTLY, FC_STRING_CONTAINS, FC_STRING_DOES_NOT_CONTAIN, + FC_STRING_ENDS_WITH, FC_STRING_EXACTLY, FC_STRING_NOT_EXACTLY, + FC_STRING_STARTS_WITH, FC_STRING_CONTAINS_CASE_INSENSITIVE + ] + + +def get_default_df_formats(column_definitions: Optional[List[ColumnDefinitions]], dfs: List[pd.DataFrame]) -> Optional[List[DataframeFormat]]: + + if column_definitions is None: + # If no column_definitions are provided, end early + return None + + if len(column_definitions) > len(dfs): + raise ValueError(f"column_definitions has formatting for {len(column_definitions)} dataframes, but only {len(dfs)} dataframes are provided.") + + df_formats = [] + + for sheet_index, column_definitions_for_sheet in enumerate(column_definitions): + df = dfs[sheet_index] + + df_format: DataframeFormat = { + 'columns': {}, + 'headers': {}, + 'rows': {'even': {}, 'odd': {}}, + 'border': {}, + 'conditional_formats': [] + } + conditional_formats = [] + for column_defintion in column_definitions_for_sheet: + conditional_formats_list: List[ColumnDefinitionConditionalFormats] = column_defintion['conditional_formats'] + for conditional_format in conditional_formats_list: + + font_color = conditional_format.get('font_color', None) + background_color = conditional_format.get('background_color', None) + columns = column_defintion['columns'] + + # Validate the font_color and/or background_color is set + if font_color is None and background_color is None: + raise ValueError(f"column_definititon has invalid conditional_format rules. It must set the font_color, background_color, or both.") + + # Validate the font_color is a hex value for a color + invalid_hex_color_error_message = "The {variable} {color} set in column_definititon is not a valid hex color. It should start with '#' and be followed by the letters from a-f, A-F and/or digits from 0-9. The length of the hexadecimal color code should be either 6 or 3, excluding '#' symbol" + if font_color and not is_valid_hex_color(font_color): + raise ValueError(invalid_hex_color_error_message.format(variable="font_color", color=font_color)) + + # Validate the background_color is a hex value for a color + if background_color and not is_valid_hex_color(background_color): + raise ValueError(invalid_hex_color_error_message.format(variable="background_color", color=background_color)) + + # Validate all of the columns exist in the dataframe + non_existant_colums = [str(column) for column in columns if column not in list(df.columns)] + if len(non_existant_colums) > 0: + raise ValueError(f"column_definititon attempts to set conditional formatting on columns {', '.join(non_existant_colums)}, but {'it' if len(non_existant_colums) == 0 else 'they'} don't exist in the dataframe.") + + # Validate the filter conditions are valid + for filter in conditional_format['filters']: + if not is_valid_filter_condition(filter['condition']): + raise ValueError(f"column_definititon has invalid conditional_format rules. The condition {filter['condition']} is not a valid filter condition.") + + # Note: We do not verify that: + # 1. The filters are valid for the column type + # 2. The filters value is valid for the condition type + # because we assume that the app developer would rather the app render without the conditional formatting + # than to error, and the frontend handles these changes gracefully in the conditional formatting UI. + # Other errors, like the condition not being a valid condition supported by Mito are sheet crashing errors. + + new_conditional_format: ConditionalFormat = { + 'format_uuid': 'format_uuid_' + str(random.random()), + 'columnIDs': column_defintion['columns'], + 'filters': conditional_format['filters'], + 'invalidFilterColumnIDs': [], + 'color': font_color, + 'backgroundColor': conditional_format['background_color'] + } + + conditional_formats.append(new_conditional_format) + + df_format['conditional_formats'] = conditional_formats + df_formats.append(df_format) + + return df_formats def _get_column_id_from_header_safe( column_header: ColumnHeader,