Merge pull request #4 from xoeye/feature/strip-attrs-defaults

1.5.0: Allow stripping of attrs defaults on unstructure

Author: Peter Gaultney

CHANGES.md

@@ -1,3 +1,13 @@
+## 1.5.0
+
+New converter which will strip (recursively) all attributes which are
+equal to their attrs defaults, except for attributes annotated as
+`Literal`, which are assumed to be required.
+
+Can be called using `unstruc_strip_defaults(your_attrs_obj)`, or on a
+Cat type using the new boolean keyword argument on `unstruc`,
+`your_cats_obj.unstruc(strip_defaults=True)`.
+
 ### 1.4.1
 
 No longer assume that the `dict` methods will not be overlaid on a

README.md

@@ -199,6 +199,23 @@ features. The 3 core features are:
    a Wildcat but its underlying `dict` would be overlaid and
    inaccessible as a Wildcat.
 
+4. ### Strip `attrs` defaults on unstructure
+
+   All attributes where the value matches the default, except for
+   attributes annotated as `Literal`, can have their defaults stripped
+   recursively during unstructure.
+
+   This is accomplished via a new built-in Converter instance, and
+   does not require use of features 1-3; in fact it will work with
+   pure `attrs` classes.
+
+   This is new as of 1.5.0, is not the default behavior, and is
+   fully backwards-compatible. It is enabled by a specific call to
+   `unstruc_strip_defaults` or via a boolean keyword-only argument on
+   the mixin method, `obj.unstruc(strip_defaults=True)`.
+
+
+
 
 ## Notes on intent, compatibility, and dependencies
 

tests/test_strip_defaults.py

@@ -0,0 +1,107 @@
+from typing import Dict, Any
+from typing_extensions import Literal
+
+import attr
+from attr import Factory as fac
+
+from typecats import Cat, unstruc_strip_defaults
+
+
+def test_clean_cats_basic():
+    @Cat
+    class Clean:
+        s: str = ""
+        i: int = 8
+        lst: list = attr.Factory(list)
+
+    a = Clean()
+    assert unstruc_strip_defaults(a) == dict()
+
+    b = Clean(i=4, lst=[1])
+
+    assert unstruc_strip_defaults(b) == dict(lst=[1], i=4)
+
+
+def test_method_works_also():
+    @Cat
+    class Clean:
+        s: str = ""
+        i: int = 8
+        lst: list = attr.Factory(list)
+
+    a = Clean()
+    assert a.unstruc(strip_defaults=True) == dict()
+
+    b = Clean(i=4, lst=[1])
+
+    assert b.unstruc(strip_defaults=True) == dict(lst=[1], i=4)
+
+
+def test_clean_wildcat():
+    @Cat
+    class WC(dict):
+        s: str = ""
+        i: int = 8
+        lst: list = attr.Factory(list)
+
+    a = WC()
+    assert unstruc_strip_defaults(a) == dict()
+
+    b = WC(i=4, lst=[1])
+    b["f"] = 2
+
+    assert unstruc_strip_defaults(b) == dict(lst=[1], i=4, f=2)
+
+
+def test_clean_literal():
+    @Cat
+    class CleanLit:
+        g: int
+        s: str = ""
+        a: Literal["a"] = "a"
+
+    cl = CleanLit(g=12)
+    assert unstruc_strip_defaults(cl) == dict(a="a", g=12)
+
+
+@Cat
+class WC(dict):
+    has_s: str = ""
+    has_b: bool = False
+
+
+@Cat
+class Org(dict):
+    id: str
+    all: Literal[1] = 1
+    strip_me: Dict[str, Any] = fac(dict)
+    userClaims: WC = fac(WC)
+
+
+def test_clean_with_wildcat_underneath():
+
+    # this wildcat has a non-default value for a known key
+    od = dict(id="id1", userClaims=dict(has_s="ssss", random="a string 1"))
+    assert unstruc_strip_defaults(Org.struc(od)) == dict(od, all=1)
+
+    # this wildcat has no no-default values for known keys, but still
+    # has wildcat data and should not be stripped.
+    od = dict(id="id2", userClaims=dict(random="a string"))
+    assert unstruc_strip_defaults(Org.struc(od)) == dict(od, all=1)
+
+
+def test_works_with_pure_attrs_obj():
+    """don't need a Cat annotation to take advantage of this!"""
+
+    @attr.s(auto_attribs=True)
+    class Strippable:
+        s: str = ""
+        i: int = 8
+        lst: list = attr.Factory(list)
+
+    a = Strippable()
+    assert unstruc_strip_defaults(a) == dict()
+
+    b = Strippable(i=4, lst=[1])
+
+    assert unstruc_strip_defaults(b) == dict(lst=[1], i=4)

tests/test_wildcats.py

@@ -178,8 +178,8 @@ class WithId(dict):
 def test_wildcat_repr_no_conflicts():
     @Cat
     class WithDictMethodAttrs(dict):
-        items: ty.List[int]
-        keys: ty.Set[str]
+        items: ty.List[int]  # type: ignore
+        keys: ty.Set[str]  # type: ignore
 
     wdma = WithDictMethodAttrs([1, 2, 3], {"a", "b"})
 

typecats/__about__.py

@@ -1,4 +1,4 @@
 """typecats"""
-__version__ = "1.4.1"
+__version__ = "1.5.0"
 __author__ = "Peter Gaultney"
 __author_email__ = "pgaultney@xoi.io"

typecats/__init__.py

@@ -1,6 +1,14 @@
-from .tc import Cat, unstruc, struc, register_struc_hook, register_unstruc_hook  # noqa
+from .tc import (  # noqa
+    Cat,
+    unstruc,
+    struc,
+    try_struc,
+    register_struc_hook,
+    register_unstruc_hook,
+)
 from .wildcat import is_wildcat  # noqa
 from .patch import patch_converter_for_typecats  # noqa
+from .strip_defaults import unstruc_strip_defaults  # noqa
 
 from .__about__ import __version__
 

typecats/cats_mypy_plugin.py

@@ -7,6 +7,7 @@
     Var,
     Argument,
     ARG_POS,
+    ARG_NAMED_OPT,
     FuncDef,
     PassStmt,
     Block,
@@ -75,6 +76,7 @@ def add_struc_and_unstruc_to_classdefcontext(cls_def_ctx: ClassDefContext):
 
             dict_type = cls_def_ctx.api.named_type("__builtins__.dict")
             str_type = cls_def_ctx.api.named_type("__builtins__.str")
+            bool_type = cls_def_ctx.api.named_type("__builtins__.bool")
             api = cls_def_ctx.api
             implicit_any = AnyType(TypeOfAny.special_form)
             mapping = api.lookup_fully_qualified_or_none("typing.Mapping")
@@ -125,7 +127,19 @@ def add_struc_and_unstruc_to_classdefcontext(cls_def_ctx: ClassDefContext):
                         maybe_cat_return_type,
                     )
                 if UNSTRUCTURE_NAME not in info.names:
-                    add_method(cls_def_ctx, UNSTRUCTURE_NAME, [], dict_type)
+                    add_method(
+                        cls_def_ctx,
+                        UNSTRUCTURE_NAME,
+                        [
+                            Argument(
+                                Var("strip_defaults", bool_type),
+                                bool_type,
+                                None,
+                                ARG_NAMED_OPT,
+                            )
+                        ],
+                        dict_type,
+                    )
 
         if fullname == CAT_PATH:
             return add_struc_and_unstruc_to_classdefcontext

typecats/strip_defaults.py

@@ -0,0 +1,96 @@
+import typing as ty
+from functools import lru_cache
+from typing_extensions import Literal
+
+import attr
+from cattr.converters import _is_attrs_class, Converter
+
+from .patch import TypecatsCattrPatch
+
+C = ty.TypeVar("C")
+
+
+_MISSING = object()
+
+
+@lru_cache(128)
+def _get_factory_default(_attr):
+    return _attr.default.factory()
+
+
+def _get_attr_default_value(attribute) -> ty.Any:
+    if not isinstance(attribute.default, attr.Factory):  # type: ignore
+        return attribute.default
+    return _get_factory_default(attribute)
+
+
+def _strip_attr_defaults(
+    attrs_type: ty.Type, m: ty.Mapping[str, ty.Any]
+) -> ty.Dict[str, ty.Any]:
+    """The idea here is that when you are using pure dicts, a key can be
+    missing to indicate absence.  But if you're dealing with typed
+    objects, that's not possible since all keys are always present.  So
+    the only 'reasonable' way to determine what a 'union' means in a
+    class-based world is to prefer non-default values to default values at
+    all times, which attrs can tell us about.
+    """
+    attr_defaults_by_name = {
+        attribute.name: _get_attr_default_value(attribute)
+        for attribute in attrs_type.__attrs_attrs__
+    }
+    return {
+        k: v
+        for k, v in m.items()
+        if k not in attr_defaults_by_name or v != attr_defaults_by_name[k]
+    }
+
+
+def _get_names_of_defaulted_attrs(
+    unstructured: ty.Dict[str, ty.Any], attrs_obj: ty.Any
+) -> ty.Set[str]:
+    """We must provide the partially unstructured version of the object in
+    case this is a wildcat with keys that aren't defined as part of
+    the Attr type.
+    """
+    res: ty.Set[str] = set()
+    for _attr in attrs_obj.__attrs_attrs__:
+        if getattr(_attr.type, "__origin__", None) is Literal:
+            # don't strip attributes annotated as Literals - they're requirements, not "defaults"
+            continue
+        if unstructured.get(_attr.name, _MISSING) == _get_attr_default_value(_attr):
+            res.add(_attr.name)
+    return res
+
+
+def _strip_attrs_defaults(
+    unstructured_but_unclean: ty.Any, obj_to_unstructure: ty.Any
+) -> ty.Any:
+    if _is_attrs_class(obj_to_unstructure.__class__):
+        keys_to_strip = _get_names_of_defaulted_attrs(
+            unstructured_but_unclean, obj_to_unstructure
+        )
+        return {
+            k: v for k, v in unstructured_but_unclean.items() if k not in keys_to_strip
+        }
+    return unstructured_but_unclean
+
+
+class StripAttrsDefaultsOnUnstructurePatch(TypecatsCattrPatch):
+    def unstructure_patch(
+        self, original_handler: ty.Callable, obj_to_unstructure: ty.Any
+    ) -> ty.Any:
+        rv = super().unstructure_patch(original_handler, obj_to_unstructure)
+        return _strip_attrs_defaults(rv, obj_to_unstructure)
+
+
+_STRIP_DEFAULTS_CONVERTER = Converter()  # create converter
+__PATCH = StripAttrsDefaultsOnUnstructurePatch(_STRIP_DEFAULTS_CONVERTER)  # patch it
+
+
+def get_stripping_converter() -> Converter:
+    return _STRIP_DEFAULTS_CONVERTER
+
+
+def unstruc_strip_defaults(obj: ty.Any) -> ty.Any:
+    """This is the only thing anyone outside needs to worry about"""
+    return _STRIP_DEFAULTS_CONVERTER.unstructure(obj)

typecats/tc.py

@@ -13,6 +13,7 @@
 )
 from .types import C, StrucInput, UnstrucOutput, CommonStructuringExceptions
 from .patch import patch_converter_for_typecats, log_structure_exception
+from .strip_defaults import get_stripping_converter
 
 
 class TypeCat:
@@ -55,9 +56,14 @@ def _struc(cl: ty.Type[C], obj: StrucInput) -> C:
     return _struc
 
 
-def make_unstruc(converter: cattr.Converter):
-    def _unstruc(obj: ty.Any) -> ty.Any:
+def make_unstruc(
+    converter: cattr.Converter,
+    stripping_converter: cattr.Converter = get_stripping_converter(),
+):
+    def _unstruc(obj: ty.Any, *, strip_defaults: bool = False) -> ty.Any:
         """A wrapper for cattrs unstructure using the internal converter"""
+        if strip_defaults:
+            return stripping_converter.unstructure(obj)
         return converter.unstructure(obj)
 
     return _unstruc
@@ -199,13 +205,15 @@ def try_struc_cat(d: ty.Optional[StrucInput]) -> ty.Optional[C]:
 
 
 def set_unstruc_converter(
-    cls: ty.Type[C], converter: cattr.Converter = _TYPECATS_DEFAULT_CONVERTER
+    cls: ty.Type[C],
+    converter: cattr.Converter = _TYPECATS_DEFAULT_CONVERTER,
+    strip_defaults_converter: cattr.Converter = get_stripping_converter(),
 ):
     """If you want to change your mind about the built-in Converter that
     is meant to run when you call the object method YourCatObj.unstruc(), you
     can reset it here. By default, it is defined by the converter
     keyword argument on the Cat decorator.
 
     """
-    _unstruc = make_unstruc(converter)
+    _unstruc = make_unstruc(converter, strip_defaults_converter)
     setattr(cls, UNSTRUCTURE_NAME, _unstruc)