diff --git a/python_modules/dagster/dagster/utils/__init__.py b/python_modules/dagster/dagster/utils/__init__.py index e9823288434c4..3c9174ad8a420 100644 --- a/python_modules/dagster/dagster/utils/__init__.py +++ b/python_modules/dagster/dagster/utils/__init__.py @@ -492,3 +492,7 @@ def compose(*args): # reduce using functional composition over all the arguments, with the identity function as # initializer return functools.reduce(lambda f, g: lambda x: f(g(x)), args, lambda x: x) + + +def dict_without_keys(ddict, *keys): + return {key: value for key, value in ddict.items() if key not in set(keys)} diff --git a/python_modules/libraries/dagster-dask/dagster_dask/data_frame.py b/python_modules/libraries/dagster-dask/dagster_dask/data_frame.py index 32a43184a2a44..e8ee438b97ea1 100644 --- a/python_modules/libraries/dagster-dask/dagster_dask/data_frame.py +++ b/python_modules/libraries/dagster-dask/dagster_dask/data_frame.py @@ -19,6 +19,7 @@ dagster_type_materializer, ) from dagster.config.field_utils import Selector +from dagster.utils import dict_without_keys WriteCompressionTextOptions = Enum( 'WriteCompressionText', [EnumValue('gzip'), EnumValue('bz2'), EnumValue('xz'),], @@ -29,10 +30,6 @@ ) -def dict_without_keys(ddict, *keys): - return {key: value for key, value in ddict.items() if key not in set(keys)} - - @dagster_type_materializer( Selector( { diff --git a/python_modules/libraries/dagster-pandas/dagster_pandas/data_frame.py b/python_modules/libraries/dagster-pandas/dagster_pandas/data_frame.py index 775b04d1ff7ad..07940756154c0 100644 --- a/python_modules/libraries/dagster-pandas/dagster_pandas/data_frame.py +++ b/python_modules/libraries/dagster-pandas/dagster_pandas/data_frame.py @@ -20,14 +20,11 @@ ) from dagster.config.field_utils import Selector from dagster.utils.backcompat import canonicalize_backcompat_args +from dagster.utils import dict_without_keys CONSTRAINT_BLACKLIST = {ColumnDTypeFnConstraint, ColumnDTypeInSetConstraint} -def dict_without_keys(ddict, *keys): - return {key: value for key, value in ddict.items() if key not in set(keys)} - - @dagster_type_materializer( Selector( { diff --git a/python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py b/python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py index 444fc9d5b37d1..60aee0336e321 100644 --- a/python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py +++ b/python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py @@ -21,6 +21,7 @@ from dagster.config.field_utils import Selector from dagster.core.storage.system_storage import fs_intermediate_storage, fs_system_storage from dagster.core.storage.type_storage import TypeStoragePlugin +from dagster.utils import dict_without_keys WriteModeOptions = Enum( 'WriteMode', @@ -75,10 +76,6 @@ ) -def dict_without_keys(ddict, *keys): - return {key: value for key, value in ddict.items() if key not in set(keys)} - - @dagster_type_materializer( Selector( { diff --git a/python_modules/libraries/dagster-pyspark/dagster_pyspark_tests/test_types.py b/python_modules/libraries/dagster-pyspark/dagster_pyspark_tests/test_types.py index fb5eaf80357bc..b7b8f3946d7bb 100644 --- a/python_modules/libraries/dagster-pyspark/dagster_pyspark_tests/test_types.py +++ b/python_modules/libraries/dagster-pyspark/dagster_pyspark_tests/test_types.py @@ -13,6 +13,7 @@ file_relative_path, solid, ) +from dagster.utils import dict_without_keys from dagster.utils.test import get_temp_dir spark = SparkSession.builder.getOrCreate() @@ -28,10 +29,6 @@ ] -def dict_without_keys(ddict, *keys): - return {key: value for key, value in ddict.items() if key not in set(keys)} - - def create_pyspark_df(): data = [Row(_c0=str(i), _c1=str(i)) for i in range(100)] return spark.createDataFrame(data)