move dict_without_keys to dagster.utils

dagster-io · Aug 21, 2020 · bf4b519 · bf4b519
1 parent c5284f5
commit bf4b519
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 16 deletions.
diff --git a/python_modules/dagster/dagster/utils/__init__.py b/python_modules/dagster/dagster/utils/__init__.py
@@ -492,3 +492,7 @@ def compose(*args):
     # reduce using functional composition over all the arguments, with the identity function as
     # initializer
     return functools.reduce(lambda f, g: lambda x: f(g(x)), args, lambda x: x)
+
+
+def dict_without_keys(ddict, *keys):
+    return {key: value for key, value in ddict.items() if key not in set(keys)}
diff --git a/python_modules/libraries/dagster-dask/dagster_dask/data_frame.py b/python_modules/libraries/dagster-dask/dagster_dask/data_frame.py
@@ -19,6 +19,7 @@
     dagster_type_materializer,
 )
 from dagster.config.field_utils import Selector
+from dagster.utils import dict_without_keys
 
 WriteCompressionTextOptions = Enum(
     'WriteCompressionText', [EnumValue('gzip'), EnumValue('bz2'), EnumValue('xz'),],
@@ -29,10 +30,6 @@
 )
 
 
-def dict_without_keys(ddict, *keys):
-    return {key: value for key, value in ddict.items() if key not in set(keys)}
-
-
 @dagster_type_materializer(
     Selector(
         {

diff --git a/python_modules/libraries/dagster-pandas/dagster_pandas/data_frame.py b/python_modules/libraries/dagster-pandas/dagster_pandas/data_frame.py
@@ -20,14 +20,11 @@
 )
 from dagster.config.field_utils import Selector
 from dagster.utils.backcompat import canonicalize_backcompat_args
+from dagster.utils import dict_without_keys
 
 CONSTRAINT_BLACKLIST = {ColumnDTypeFnConstraint, ColumnDTypeInSetConstraint}
 
 
-def dict_without_keys(ddict, *keys):
-    return {key: value for key, value in ddict.items() if key not in set(keys)}
-
-
 @dagster_type_materializer(
     Selector(
         {

diff --git a/python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py b/python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py
@@ -21,6 +21,7 @@
 from dagster.config.field_utils import Selector
 from dagster.core.storage.system_storage import fs_intermediate_storage, fs_system_storage
 from dagster.core.storage.type_storage import TypeStoragePlugin
+from dagster.utils import dict_without_keys
 
 WriteModeOptions = Enum(
     'WriteMode',
@@ -75,10 +76,6 @@
 )
 
 
-def dict_without_keys(ddict, *keys):
-    return {key: value for key, value in ddict.items() if key not in set(keys)}
-
-
 @dagster_type_materializer(
     Selector(
         {

diff --git a/python_modules/libraries/dagster-pyspark/dagster_pyspark_tests/test_types.py b/python_modules/libraries/dagster-pyspark/dagster_pyspark_tests/test_types.py
@@ -13,6 +13,7 @@
     file_relative_path,
     solid,
 )
+from dagster.utils import dict_without_keys
 from dagster.utils.test import get_temp_dir
 
 spark = SparkSession.builder.getOrCreate()
@@ -28,10 +29,6 @@
 ]
 
 
-def dict_without_keys(ddict, *keys):
-    return {key: value for key, value in ddict.items() if key not in set(keys)}
-
-
 def create_pyspark_df():
     data = [Row(_c0=str(i), _c1=str(i)) for i in range(100)]
     return spark.createDataFrame(data)