handle deepObject and explode: true (#971)

* first implementation draft

* gitignore virtualenv

* use isinstance instead of type function

* fix tests

* remove unused function

* move object parsing to uri_parsing.py

* remove not needed import

* only test for OpenAPI

* remove not needed import

* make it work for other cases again

* flake8 fixes

* python2.7 fixes

* isort fix

* address code review comments

* remove for loop and address other comments

* remove not needed abstract function

* move array unnesting into uri_parsing

* make nested arrays possible

* style fixes

* style fixes

* test other data types

* comment and simplify function

* WIP: start additionalProperties test

* test additionalProperties

* remove uneccessary exception

* set default values

* set default values also in response

* flake8 fixes

* fix test

* use suggestions from dtkav's branch

* fix tests partially

* fix tests partially

* fix tests

* fix tests

* add comments for clarity

Author: drummerwolli

.gitignore

@@ -11,3 +11,4 @@ htmlcov/
 *.swp
 .tox/
 .idea/
+venv/

connexion/decorators/uri_parsing.py

@@ -2,9 +2,11 @@
 import abc
 import functools
 import logging
+import re
 
 import six
 
+from ..utils import create_empty_dict_from_list
 from .decorator import BaseDecorator
 
 logger = logging.getLogger('connexion.decorators.uri_parsing')
@@ -96,8 +98,23 @@ def resolve_params(self, params, _in):
         """
         resolved_param = {}
         for k, values in params.items():
+            # extract the dict keys if specified with style: deepObject and explode: true
+            # according to https://swagger.io/docs/specification/serialization/#query
+            dict_keys = re.findall(r'\[(\w+)\]', k)
+            if dict_keys:
+                k = k.split("[", 1)[0]
+                param_defn = self.param_defns.get(k)
+                if param_defn and param_defn.get('style', None) == 'deepObject' and param_defn.get('explode', False):
+                    param_schema = self.param_schemas.get(k)
+                    if isinstance(values, list) and len(values) == 1 and param_schema['type'] != 'array':
+                        values = values[0]
+                    resolved_param.setdefault(k, {})
+                    resolved_param[k].update(create_empty_dict_from_list(dict_keys, {}, values))
+                    continue
+
             param_defn = self.param_defns.get(k)
             param_schema = self.param_schemas.get(k)
+
             if not (param_defn or param_schema):
                 # rely on validation
                 resolved_param[k] = values
@@ -115,8 +132,21 @@ def resolve_params(self, params, _in):
             else:
                 resolved_param[k] = values[-1]
 
+        # set defaults if values have not been set yet
+        resolved_param = self.set_default_values(resolved_param, self.param_schemas)
+
         return resolved_param
 
+    def set_default_values(self, _dict, _properties):
+        """set recursively default values in objects/dicts"""
+        for p_id, property in _properties.items():
+            if 'default' in property and p_id not in _dict:
+                _dict[p_id] = property['default']
+            elif property.get('type', False) == 'object' and 'properties' in property:
+                _dict.setdefault(p_id, {})
+                _dict[p_id] = self.set_default_values(_dict[p_id], property['properties'])
+        return _dict
+
     def __call__(self, function):
         """
         :type function: types.FunctionType

connexion/decorators/validation.py

@@ -19,7 +19,8 @@
 TYPE_MAP = {
     'integer': int,
     'number': float,
-    'boolean': boolean
+    'boolean': boolean,
+    'object': dict
 }
 
 
@@ -63,6 +64,21 @@ def make_type(value, type_literal):
                 converted = v
             converted_params.append(converted)
         return converted_params
+    elif param_type == 'object':
+        if param_schema.get('properties'):
+            def cast_leaves(d, schema):
+                if type(d) is not dict:
+                    try:
+                        return make_type(d, schema['type'])
+                    except (ValueError, TypeError):
+                        return d
+                for k, v in d.items():
+                    if k in schema['properties']:
+                        d[k] = cast_leaves(v, schema['properties'][k])
+                return d
+
+            return cast_leaves(value, param_schema)
+        return value
     else:
         try:
             return make_type(value, param_type)

connexion/operations/abstract.py

@@ -199,7 +199,7 @@ def _query_args_helper(self, query_defns, query_arguments,
                     logger.error("Function argument '{}' not defined in specification".format(key))
                 else:
                     logger.debug('%s is a %s', key, query_defn)
-                    res[key] = self._get_val_from_param(value, query_defn)
+                    res.update({key: self._get_val_from_param(value, query_defn)})
         return res
 
     @abc.abstractmethod

connexion/operations/openapi.py

@@ -322,5 +322,16 @@ def _get_val_from_param(self, value, query_defn):
 
         if query_schema["type"] == "array":
             return [make_type(part, query_schema["items"]["type"]) for part in value]
+        elif query_schema["type"] == "object" and 'properties' in query_schema:
+            return_dict = {}
+            for prop_key in query_schema['properties'].keys():
+                prop_value = value.get(prop_key, None)
+                if prop_value is not None:  # False is a valid value for boolean values
+                    try:
+                        return_dict[prop_key] = make_type(value[prop_key],
+                                                          query_schema['properties'][prop_key]['type'])
+                    except (KeyError, TypeError):
+                        return value
+            return return_dict
         else:
             return make_type(value, query_schema["type"])

connexion/utils.py

@@ -1,5 +1,6 @@
 import functools
 import importlib
+import re
 
 import six
 import yaml
@@ -253,3 +254,12 @@ def ignore_aliases(self, *args):
     yaml.representer.SafeRepresenter.represent_scalar = my_represent_scalar
 
     return yaml.dump(openapi, allow_unicode=True, Dumper=NoAnchorDumper)
+
+
+def create_empty_dict_from_list(_list, _dict, _end_value):
+    """create from ['foo', 'bar'] a dict like {'foo': {'bar': {}}} recursively. needed for converting query params"""
+    current_key = _list.pop(0)
+    if _list:
+        return {current_key: create_empty_dict_from_list(_list, _dict, _end_value)}
+    else:
+        return {current_key: _end_value}

tests/api/test_responses.py

@@ -108,6 +108,42 @@ def test_empty(simple_app):
     assert not response.data
 
 
+def test_exploded_deep_object_param_endpoint_openapi_simple(simple_openapi_app):
+    app_client = simple_openapi_app.app.test_client()
+
+    response = app_client.get('/v1.0/exploded-deep-object-param?id[foo]=bar&id[foofoo]=barbar')  # type: flask.Response
+    assert response.status_code == 200
+    response_data = json.loads(response.data.decode('utf-8', 'replace'))
+    assert response_data == {'foo': 'bar', 'foo4': 'blubb'}
+
+
+def test_exploded_deep_object_param_endpoint_openapi_multiple_data_types(simple_openapi_app):
+    app_client = simple_openapi_app.app.test_client()
+
+    response = app_client.get('/v1.0/exploded-deep-object-param?id[foo]=bar&id[fooint]=2&id[fooboo]=false')  # type: flask.Response
+    assert response.status_code == 200
+    response_data = json.loads(response.data.decode('utf-8', 'replace'))
+    assert response_data == {'foo': 'bar', 'fooint': 2, 'fooboo': False, 'foo4': 'blubb'}
+
+
+def test_exploded_deep_object_param_endpoint_openapi_additional_properties(simple_openapi_app):
+    app_client = simple_openapi_app.app.test_client()
+
+    response = app_client.get('/v1.0/exploded-deep-object-param-additional-properties?id[foo]=bar&id[fooint]=2')  # type: flask.Response
+    assert response.status_code == 200
+    response_data = json.loads(response.data.decode('utf-8', 'replace'))
+    assert response_data == {'foo': 'bar', 'fooint': '2'}
+
+
+def test_nested_exploded_deep_object_param_endpoint_openapi(simple_openapi_app):
+    app_client = simple_openapi_app.app.test_client()
+
+    response = app_client.get('/v1.0/nested-exploded-deep-object-param?id[foo][foo2]=bar&id[foofoo]=barbar')  # type: flask.Response
+    assert response.status_code == 200
+    response_data = json.loads(response.data.decode('utf-8', 'replace'))
+    assert response_data == {'foo': {'foo2': 'bar', 'foo3': 'blubb'}, 'foofoo': 'barbar'}
+
+
 def test_redirect_endpoint(simple_app):
     app_client = simple_app.app.test_client()
     resp = app_client.get('/v1.0/test-redirect-endpoint')

tests/conftest.py

@@ -12,7 +12,9 @@
 TEST_FOLDER = pathlib.Path(__file__).parent
 FIXTURES_FOLDER = TEST_FOLDER / 'fixtures'
 SPEC_FOLDER = TEST_FOLDER / "fakeapi"
-SPECS = ["swagger.yaml", "openapi.yaml"]
+OPENAPI2_SPEC = ["swagger.yaml"]
+OPENAPI3_SPEC = ["openapi.yaml"]
+SPECS = OPENAPI2_SPEC + OPENAPI3_SPEC
 
 
 class FakeResponse(object):
@@ -116,6 +118,11 @@ def simple_app(request):
     return build_app_from_fixture('simple', request.param, validate_responses=True)
 
 
+@pytest.fixture(scope="session", params=OPENAPI3_SPEC)
+def simple_openapi_app(request):
+    return build_app_from_fixture('simple', request.param, validate_responses=True)
+
+
 @pytest.fixture(scope="session", params=SPECS)
 def snake_case_app(request):
     return build_app_from_fixture('snake_case', request.param,

tests/decorators/test_validation.py

@@ -43,6 +43,13 @@ def test_get_nullable_parameter():
     assert result is None
 
 
+def test_get_explodable_object_parameter():
+    param = {'schema': {'type': 'object', 'additionalProperties': True},
+             'required': True, 'name': 'foo', 'style': 'deepObject', 'explode': True}
+    result = ParameterValidator.validate_parameter('query', {'bar': 1}, param)
+    assert result is None
+
+
 def test_invalid_type(monkeypatch):
     logger = MagicMock()
     monkeypatch.setattr('connexion.decorators.validation.logger', logger)

tests/fakeapi/hello.py

@@ -318,6 +318,18 @@ def test_required_param(simple):
     return simple
 
 
+def test_exploded_deep_object_param(id):
+    return id
+
+
+def test_nested_exploded_deep_object_param(id):
+    return id
+
+
+def test_exploded_deep_object_param_additional_properties(id):
+    return id
+
+
 def test_redirect_endpoint():
     headers = {'Location': 'http://www.google.com/'}
     return '', 302, headers

tests/fixtures/simple/openapi.yaml

@@ -150,6 +150,104 @@ paths:
       responses:
         '204':
           description: empty
+  /exploded-deep-object-param:
+    get:
+      summary: Returns dict response
+      description: Returns dict response
+      operationId: fakeapi.hello.test_exploded_deep_object_param
+      parameters:
+        - name: id
+          required: true
+          in: query
+          style: deepObject
+          explode: true
+          schema:
+            type: object
+            properties:
+              foo:
+                type: string
+              fooint:
+                type: integer
+              fooboo:
+                type: boolean
+              foo4:
+                type: string
+                default: blubb
+      responses:
+        '200':
+          description: object response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  foo:
+                    type: string
+                  foo4:
+                    type: string
+  /exploded-deep-object-param-additional-properties:
+    get:
+      summary: Returns dict response with flexible properties
+      description: Returns dict response with flexible properties
+      operationId: fakeapi.hello.test_exploded_deep_object_param_additional_properties
+      parameters:
+        - name: id
+          required: false
+          in: query
+          style: deepObject
+          explode: true
+          schema:
+            type: object
+            additionalProperties:
+              type: string
+      responses:
+        '200':
+          description: object response
+          content:
+            application/json:
+              schema:
+                type: object
+                additionalProperties:
+                  type: string
+  /nested-exploded-deep-object-param:
+    get:
+      summary: Returns nested dict response
+      description: Returns nested dict response
+      operationId: fakeapi.hello.test_nested_exploded_deep_object_param
+      parameters:
+        - name: id
+          required: true
+          in: query
+          style: deepObject
+          explode: true
+          schema:
+            type: object
+            properties:
+              foo:
+                type: object
+                properties:
+                  foo2:
+                    type: string
+                  foo3:
+                    type: string
+                    default: blubb
+              foofoo:
+                type: string
+      responses:
+        '200':
+          description: object response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  foo:
+                    type: object
+                    properties:
+                      foo2:
+                        type: string
+                      foo3:
+                        type: string
   /test-redirect-endpoint:
     get:
       summary: Tests handlers returning flask.Response objects