[Python] fix model to dict (#11234)

* fix model to dict

* add unit test

* add unit test

* update sample files

modules/openapi-generator/src/main/resources/python/model_utils.mustache

@@ -1340,6 +1340,7 @@ def model_to_dict(model_instance, serialize=True):
             attribute_map
     """
     result = {}
+    extract_item = lambda item: (item[0], model_to_dict(item[1], serialize=serialize)) if hasattr(item[1], '_data_store') else item
 
     model_instances = [model_instance]
     if model_instance._composed_schemas:
@@ -1369,14 +1370,17 @@ def model_to_dict(model_instance, serialize=True):
                            res.append(v)
                        elif isinstance(v, ModelSimple):
                            res.append(v.value)
+                       elif isinstance(v, dict):
+                           res.append(dict(map(
+                               extract_item,
+                               v.items()
+                           )))
                        else:
                            res.append(model_to_dict(v, serialize=serialize))
                    result[attr] = res
             elif isinstance(value, dict):
                 result[attr] = dict(map(
-                    lambda item: (item[0],
-                                  model_to_dict(item[1], serialize=serialize))
-                    if hasattr(item[1], '_data_store') else item,
+                    extract_item,
                     value.items()
                 ))
             elif isinstance(value, ModelSimple):

modules/openapi-generator/src/test/resources/3_0/python/petstore-with-fake-endpoints-models-for-testing-with-http-signature.yaml

@@ -2495,3 +2495,12 @@ components:
       type: boolean
       enum:
         - true
+    FooObject:
+      type: object
+      properties:
+        prop1:
+          type: array
+          items:
+            type: object
+        prop2:
+          type: object

samples/client/petstore/python/petstore_api/model_utils.py

@@ -1657,6 +1657,7 @@ def model_to_dict(model_instance, serialize=True):
             attribute_map
     """
     result = {}
+    extract_item = lambda item: (item[0], model_to_dict(item[1], serialize=serialize)) if hasattr(item[1], '_data_store') else item
 
     model_instances = [model_instance]
     if model_instance._composed_schemas:
@@ -1686,14 +1687,17 @@ def model_to_dict(model_instance, serialize=True):
                            res.append(v)
                        elif isinstance(v, ModelSimple):
                            res.append(v.value)
+                       elif isinstance(v, dict):
+                           res.append(dict(map(
+                               extract_item,
+                               v.items()
+                           )))
                        else:
                            res.append(model_to_dict(v, serialize=serialize))
                    result[attr] = res
             elif isinstance(value, dict):
                 result[attr] = dict(map(
-                    lambda item: (item[0],
-                                  model_to_dict(item[1], serialize=serialize))
-                    if hasattr(item[1], '_data_store') else item,
+                    extract_item,
                     value.items()
                 ))
             elif isinstance(value, ModelSimple):

samples/client/petstore/python_disallowAdditionalPropertiesIfNotPresent/petstore_api/model_utils.py

@@ -1657,6 +1657,7 @@ def model_to_dict(model_instance, serialize=True):
             attribute_map
     """
     result = {}
+    extract_item = lambda item: (item[0], model_to_dict(item[1], serialize=serialize)) if hasattr(item[1], '_data_store') else item
 
     model_instances = [model_instance]
     if model_instance._composed_schemas:
@@ -1686,14 +1687,17 @@ def model_to_dict(model_instance, serialize=True):
                            res.append(v)
                        elif isinstance(v, ModelSimple):
                            res.append(v.value)
+                       elif isinstance(v, dict):
+                           res.append(dict(map(
+                               extract_item,
+                               v.items()
+                           )))
                        else:
                            res.append(model_to_dict(v, serialize=serialize))
                    result[attr] = res
             elif isinstance(value, dict):
                 result[attr] = dict(map(
-                    lambda item: (item[0],
-                                  model_to_dict(item[1], serialize=serialize))
-                    if hasattr(item[1], '_data_store') else item,
+                    extract_item,
                     value.items()
                 ))
             elif isinstance(value, ModelSimple):

samples/openapi3/client/extensions/x-auth-id-alias/python/x_auth_id_alias/model_utils.py

@@ -1657,6 +1657,7 @@ def model_to_dict(model_instance, serialize=True):
             attribute_map
     """
     result = {}
+    extract_item = lambda item: (item[0], model_to_dict(item[1], serialize=serialize)) if hasattr(item[1], '_data_store') else item
 
     model_instances = [model_instance]
     if model_instance._composed_schemas:
@@ -1686,14 +1687,17 @@ def model_to_dict(model_instance, serialize=True):
                            res.append(v)
                        elif isinstance(v, ModelSimple):
                            res.append(v.value)
+                       elif isinstance(v, dict):
+                           res.append(dict(map(
+                               extract_item,
+                               v.items()
+                           )))
                        else:
                            res.append(model_to_dict(v, serialize=serialize))
                    result[attr] = res
             elif isinstance(value, dict):
                 result[attr] = dict(map(
-                    lambda item: (item[0],
-                                  model_to_dict(item[1], serialize=serialize))
-                    if hasattr(item[1], '_data_store') else item,
+                    extract_item,
                     value.items()
                 ))
             elif isinstance(value, ModelSimple):

samples/openapi3/client/features/dynamic-servers/python/dynamic_servers/model_utils.py

@@ -1657,6 +1657,7 @@ def model_to_dict(model_instance, serialize=True):
             attribute_map
     """
     result = {}
+    extract_item = lambda item: (item[0], model_to_dict(item[1], serialize=serialize)) if hasattr(item[1], '_data_store') else item
 
     model_instances = [model_instance]
     if model_instance._composed_schemas:
@@ -1686,14 +1687,17 @@ def model_to_dict(model_instance, serialize=True):
                            res.append(v)
                        elif isinstance(v, ModelSimple):
                            res.append(v.value)
+                       elif isinstance(v, dict):
+                           res.append(dict(map(
+                               extract_item,
+                               v.items()
+                           )))
                        else:
                            res.append(model_to_dict(v, serialize=serialize))
                    result[attr] = res
             elif isinstance(value, dict):
                 result[attr] = dict(map(
-                    lambda item: (item[0],
-                                  model_to_dict(item[1], serialize=serialize))
-                    if hasattr(item[1], '_data_store') else item,
+                    extract_item,
                     value.items()
                 ))
             elif isinstance(value, ModelSimple):

samples/openapi3/client/petstore/python/.openapi-generator/FILES

@@ -45,6 +45,7 @@ docs/FakePostInlineAdditionalPropertiesPayloadArrayData.md
 docs/File.md
 docs/FileSchemaTestClass.md
 docs/Foo.md
+docs/FooObject.md
 docs/FormatTest.md
 docs/Fruit.md
 docs/FruitReq.md
@@ -158,6 +159,7 @@ petstore_api/model/fake_post_inline_additional_properties_payload_array_data.py
 petstore_api/model/file.py
 petstore_api/model/file_schema_test_class.py
 petstore_api/model/foo.py
+petstore_api/model/foo_object.py
 petstore_api/model/format_test.py
 petstore_api/model/fruit.py
 petstore_api/model/fruit_req.py

samples/openapi3/client/petstore/python/README.md

@@ -174,6 +174,7 @@ Class | Method | HTTP request | Description
  - [File](docs/File.md)
  - [FileSchemaTestClass](docs/FileSchemaTestClass.md)
  - [Foo](docs/Foo.md)
+ - [FooObject](docs/FooObject.md)
  - [FormatTest](docs/FormatTest.md)
  - [Fruit](docs/Fruit.md)
  - [FruitReq](docs/FruitReq.md)

samples/openapi3/client/petstore/python/docs/FooObject.md

@@ -0,0 +1,13 @@
+# FooObject
+
+
+## Properties
+Name | Type | Description | Notes
+------------ | ------------- | ------------- | -------------
+**prop1** | **[{str: (bool, date, datetime, dict, float, int, list, str, none_type)}]** |  | [optional] 
+**prop2** | **{str: (bool, date, datetime, dict, float, int, list, str, none_type)}** |  | [optional] 
+**any string name** | **bool, date, datetime, dict, float, int, list, str, none_type** | any string name can be used but the value must be the correct type | [optional]
+
+[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+
+

samples/openapi3/client/petstore/python/petstore_api/model/foo_object.py

@@ -0,0 +1,259 @@
+"""
+    OpenAPI Petstore
+
+    This spec is mainly for testing Petstore server and contains fake endpoints, models. Please do not use this for any other purpose. Special characters: \" \\  # noqa: E501
+
+    The version of the OpenAPI document: 1.0.0
+    Generated by: https://openapi-generator.tech
+"""
+
+
+import re  # noqa: F401
+import sys  # noqa: F401
+
+from petstore_api.model_utils import (  # noqa: F401
+    ApiTypeError,
+    ModelComposed,
+    ModelNormal,
+    ModelSimple,
+    cached_property,
+    change_keys_js_to_python,
+    convert_js_args_to_python_args,
+    date,
+    datetime,
+    file_type,
+    none_type,
+    validate_get_composed_info,
+    OpenApiModel
+)
+from petstore_api.exceptions import ApiAttributeError
+
+
+
+class FooObject(ModelNormal):
+    """NOTE: This class is auto generated by OpenAPI Generator.
+    Ref: https://openapi-generator.tech
+
+    Do not edit the class manually.
+
+    Attributes:
+      allowed_values (dict): The key is the tuple path to the attribute
+          and the for var_name this is (var_name,). The value is a dict
+          with a capitalized key describing the allowed value and an allowed
+          value. These dicts store the allowed enum values.
+      attribute_map (dict): The key is attribute name
+          and the value is json key in definition.
+      discriminator_value_class_map (dict): A dict to go from the discriminator
+          variable value to the discriminator class name.
+      validations (dict): The key is the tuple path to the attribute
+          and the for var_name this is (var_name,). The value is a dict
+          that stores validations for max_length, min_length, max_items,
+          min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum,
+          inclusive_minimum, and regex.
+      additional_properties_type (tuple): A tuple of classes accepted
+          as additional properties values.
+    """
+
+    allowed_values = {
+    }
+
+    validations = {
+    }
+
+    @cached_property
+    def additional_properties_type():
+        """
+        This must be a method because a model may have properties that are
+        of type self, this must run after the class is loaded
+        """
+        return (bool, date, datetime, dict, float, int, list, str, none_type,)  # noqa: E501
+
+    _nullable = False
+
+    @cached_property
+    def openapi_types():
+        """
+        This must be a method because a model may have properties that are
+        of type self, this must run after the class is loaded
+
+        Returns
+            openapi_types (dict): The key is attribute name
+                and the value is attribute type.
+        """
+        return {
+            'prop1': ([{str: (bool, date, datetime, dict, float, int, list, str, none_type)}],),  # noqa: E501
+            'prop2': ({str: (bool, date, datetime, dict, float, int, list, str, none_type)},),  # noqa: E501
+        }
+
+    @cached_property
+    def discriminator():
+        return None
+
+
+    attribute_map = {
+        'prop1': 'prop1',  # noqa: E501
+        'prop2': 'prop2',  # noqa: E501
+    }
+
+    read_only_vars = {
+    }
+
+    _composed_schemas = {}
+
+    @classmethod
+    @convert_js_args_to_python_args
+    def _from_openapi_data(cls, *args, **kwargs):  # noqa: E501
+        """FooObject - a model defined in OpenAPI
+
+        Keyword Args:
+            _check_type (bool): if True, values for parameters in openapi_types
+                                will be type checked and a TypeError will be
+                                raised if the wrong type is input.
+                                Defaults to True
+            _path_to_item (tuple/list): This is a list of keys or values to
+                                drill down to the model in received_data
+                                when deserializing a response
+            _spec_property_naming (bool): True if the variable names in the input data
+                                are serialized names, as specified in the OpenAPI document.
+                                False if the variable names in the input data
+                                are pythonic names, e.g. snake case (default)
+            _configuration (Configuration): the instance to use when
+                                deserializing a file_type parameter.
+                                If passed, type conversion is attempted
+                                If omitted no type conversion is done.
+            _visited_composed_classes (tuple): This stores a tuple of
+                                classes that we have traveled through so that
+                                if we see that class again we will not use its
+                                discriminator again.
+                                When traveling through a discriminator, the
+                                composed schema that is
+                                is traveled through is added to this set.
+                                For example if Animal has a discriminator
+                                petType and we pass in "Dog", and the class Dog
+                                allOf includes Animal, we move through Animal
+                                once using the discriminator, and pick Dog.
+                                Then in Dog, we will make an instance of the
+                                Animal class but this time we won't travel
+                                through its discriminator because we passed in
+                                _visited_composed_classes = (Animal,)
+            prop1 ([{str: (bool, date, datetime, dict, float, int, list, str, none_type)}]): [optional]  # noqa: E501
+            prop2 ({str: (bool, date, datetime, dict, float, int, list, str, none_type)}): [optional]  # noqa: E501
+        """
+
+        _check_type = kwargs.pop('_check_type', True)
+        _spec_property_naming = kwargs.pop('_spec_property_naming', False)
+        _path_to_item = kwargs.pop('_path_to_item', ())
+        _configuration = kwargs.pop('_configuration', None)
+        _visited_composed_classes = kwargs.pop('_visited_composed_classes', ())
+
+        self = super(OpenApiModel, cls).__new__(cls)
+
+        if args:
+            raise ApiTypeError(
+                "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % (
+                    args,
+                    self.__class__.__name__,
+                ),
+                path_to_item=_path_to_item,
+                valid_classes=(self.__class__,),
+            )
+
+        self._data_store = {}
+        self._check_type = _check_type
+        self._spec_property_naming = _spec_property_naming
+        self._path_to_item = _path_to_item
+        self._configuration = _configuration
+        self._visited_composed_classes = _visited_composed_classes + (self.__class__,)
+
+        for var_name, var_value in kwargs.items():
+            if var_name not in self.attribute_map and \
+                        self._configuration is not None and \
+                        self._configuration.discard_unknown_keys and \
+                        self.additional_properties_type is None:
+                # discard variable.
+                continue
+            setattr(self, var_name, var_value)
+        return self
+
+    required_properties = set([
+        '_data_store',
+        '_check_type',
+        '_spec_property_naming',
+        '_path_to_item',
+        '_configuration',
+        '_visited_composed_classes',
+    ])
+
+    @convert_js_args_to_python_args
+    def __init__(self, *args, **kwargs):  # noqa: E501
+        """FooObject - a model defined in OpenAPI
+
+        Keyword Args:
+            _check_type (bool): if True, values for parameters in openapi_types
+                                will be type checked and a TypeError will be
+                                raised if the wrong type is input.
+                                Defaults to True
+            _path_to_item (tuple/list): This is a list of keys or values to
+                                drill down to the model in received_data
+                                when deserializing a response
+            _spec_property_naming (bool): True if the variable names in the input data
+                                are serialized names, as specified in the OpenAPI document.
+                                False if the variable names in the input data
+                                are pythonic names, e.g. snake case (default)
+            _configuration (Configuration): the instance to use when
+                                deserializing a file_type parameter.
+                                If passed, type conversion is attempted
+                                If omitted no type conversion is done.
+            _visited_composed_classes (tuple): This stores a tuple of
+                                classes that we have traveled through so that
+                                if we see that class again we will not use its
+                                discriminator again.
+                                When traveling through a discriminator, the
+                                composed schema that is
+                                is traveled through is added to this set.
+                                For example if Animal has a discriminator
+                                petType and we pass in "Dog", and the class Dog
+                                allOf includes Animal, we move through Animal
+                                once using the discriminator, and pick Dog.
+                                Then in Dog, we will make an instance of the
+                                Animal class but this time we won't travel
+                                through its discriminator because we passed in
+                                _visited_composed_classes = (Animal,)
+            prop1 ([{str: (bool, date, datetime, dict, float, int, list, str, none_type)}]): [optional]  # noqa: E501
+            prop2 ({str: (bool, date, datetime, dict, float, int, list, str, none_type)}): [optional]  # noqa: E501
+        """
+
+        _check_type = kwargs.pop('_check_type', True)
+        _spec_property_naming = kwargs.pop('_spec_property_naming', False)
+        _path_to_item = kwargs.pop('_path_to_item', ())
+        _configuration = kwargs.pop('_configuration', None)
+        _visited_composed_classes = kwargs.pop('_visited_composed_classes', ())
+
+        if args:
+            raise ApiTypeError(
+                "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % (
+                    args,
+                    self.__class__.__name__,
+                ),
+                path_to_item=_path_to_item,
+                valid_classes=(self.__class__,),
+            )
+
+        self._data_store = {}
+        self._check_type = _check_type
+        self._spec_property_naming = _spec_property_naming
+        self._path_to_item = _path_to_item
+        self._configuration = _configuration
+        self._visited_composed_classes = _visited_composed_classes + (self.__class__,)
+
+        for var_name, var_value in kwargs.items():
+            if var_name not in self.attribute_map and \
+                        self._configuration is not None and \
+                        self._configuration.discard_unknown_keys and \
+                        self.additional_properties_type is None:
+                # discard variable.
+                continue
+            setattr(self, var_name, var_value)
+            if var_name in self.read_only_vars:
+                raise ApiAttributeError(f"`{var_name}` is a read-only attribute. Use `from_openapi_data` to instantiate "
+                                     f"class with read only attributes.")

samples/openapi3/client/petstore/python/petstore_api/model_utils.py

@@ -1657,6 +1657,7 @@ def model_to_dict(model_instance, serialize=True):
             attribute_map
     """
     result = {}
+    extract_item = lambda item: (item[0], model_to_dict(item[1], serialize=serialize)) if hasattr(item[1], '_data_store') else item
 
     model_instances = [model_instance]
     if model_instance._composed_schemas:
@@ -1686,14 +1687,17 @@ def model_to_dict(model_instance, serialize=True):
                            res.append(v)
                        elif isinstance(v, ModelSimple):
                            res.append(v.value)
+                       elif isinstance(v, dict):
+                           res.append(dict(map(
+                               extract_item,
+                               v.items()
+                           )))
                        else:
                            res.append(model_to_dict(v, serialize=serialize))
                    result[attr] = res
             elif isinstance(value, dict):
                 result[attr] = dict(map(
-                    lambda item: (item[0],
-                                  model_to_dict(item[1], serialize=serialize))
-                    if hasattr(item[1], '_data_store') else item,
+                    extract_item,
                     value.items()
                 ))
             elif isinstance(value, ModelSimple):

samples/openapi3/client/petstore/python/petstore_api/models/__init__.py

@@ -48,6 +48,7 @@ from petstore_api.model.fake_post_inline_additional_properties_payload_array_dat
 from petstore_api.model.file import File
 from petstore_api.model.file_schema_test_class import FileSchemaTestClass
 from petstore_api.model.foo import Foo
+from petstore_api.model.foo_object import FooObject
 from petstore_api.model.format_test import FormatTest
 from petstore_api.model.fruit import Fruit
 from petstore_api.model.fruit_req import FruitReq

samples/openapi3/client/petstore/python/test/test_foo_object.py

@@ -0,0 +1,35 @@
+"""
+    OpenAPI Petstore
+
+    This spec is mainly for testing Petstore server and contains fake endpoints, models. Please do not use this for any other purpose. Special characters: \" \\  # noqa: E501
+
+    The version of the OpenAPI document: 1.0.0
+    Generated by: https://openapi-generator.tech
+"""
+
+
+import sys
+import unittest
+
+import petstore_api
+from petstore_api.model.foo_object import FooObject
+
+
+class TestFooObject(unittest.TestCase):
+    """FooObject unit test stubs"""
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def testFooObject(self):
+        """Test FooObject"""
+        # FIXME: construct object with mandatory attributes with example values
+        # model = FooObject()  # noqa: E501
+        pass
+
+
+if __name__ == '__main__':
+    unittest.main()

samples/openapi3/client/petstore/python/tests_manual/test_api_validation.py

@@ -163,3 +163,14 @@ class ApiClientTests(unittest.TestCase):
             }
             response = MockResponse(data=json.dumps(data))
             deserialized = api_client.deserialize(response, (format_test.FormatTest,), True)
+
+    def test_sanitize_for_serialization(self):
+        data = {
+            "prop1": [{"key1": "val1"}],
+            "prop2": {"key2": "val2"}
+        }
+        from petstore_api.model.foo_object import FooObject
+        # the property named prop1 of this model is a list of dict
+        foo_object = FooObject(prop1=data["prop1"], prop2=data["prop2"])
+        result = self.api_client.sanitize_for_serialization(foo_object)
+        self.assertEqual(data, result)