From db5941379f1b6d55075a7a8d85b63cde3e149eb8 Mon Sep 17 00:00:00 2001
From: Justin Niessner <CrshOverride@gmail.com>
Date: Sun, 26 Apr 2020 16:33:42 -0400
Subject: [PATCH] [Python] Fix Python UTF-8 Encoding Issue (#5679)

* Try decoding but don't bail on error

* Switch binary and ByteArray to bytes

* Read content type and parse appropriately

* Remove response parsing

* Remove response parsing and just return the data

* Update petshop examples w/ new generator code

* Fix copy/paste error with naming

* Update petstore examples

* Move response decoding to inside _preload_content block

* Update the clients again

* Use a raw string for the regex pattern

* Regenerate petstore clients

* Add bytes to python primitives as it's supported in 2.7 and 3

* Add bytes to the exports from model_utils

* Import bytes from model_utils

* Add conditional typing for regex pattern to match variable type

* Regenerate petstore clients

* Use read() instead of text() for asyncio

* Regenerate petstore clients

* Remove unused six import

* Regenerate petstore clients

* Add newline to kick Circle to re-run

* Remove whitespace from tox.ini

* Update more examples after ensure_updated

* Add sample updates that didn't run with the --batch flag

* Remove extra bracket in regex to remove warning

* Stop printing debug messages

* Add bytes examples to python doc generators

* Update generated FakeApi docs

* Regenerate api_client.py

* Remove print statements from generated clients

* Update bytes example in FakeApi.md. Again. I swear.

* Add yet another seemingly missing doc update

* Catch the error, decode the body, and re-throw

* Remove the updates now that the change is non-breaking

* Regenerate client

* Add bytes deserialization test

* Update exception parsing

* Add exception parsing for python-experimental

* Regenerate client with minor changes

* Revert test changes

* Regenerate model_utils.py

* Update confusing test name

* Remove bytes from mapping and examples

* Add back in the old binary/ByteArray to str mapping

* Update docs and api_client template

* Add experimental api_client changes

* Regenerate samples again

* Add Tornado handling to early return

* Try fixing Tornado python returns

* More documentation changes

* Re-generate the client code

* Remove bytes from test_format_test

* Remove more leftover bytes usages

* Switch bytes validation back to string

* Fix format_test template and regenerate

* Remove unused bytes var

* Remove bytes import from models and regenerate

* Remove bytes import from test_deserialization

* Reduce nested ifs

* Remove byte logic for now

* Regenerate client after latest changes

* Remove another bytes usage

* Regenerate after removing dangling byte string usage

* Reduce the scope of the try/catch in api_client

* Regenerate after try/catch cleanup

* Swap catch for except

* Regenerate Python client after api_client change

* Fix lint error on the generated api_client

* Add binary format test back in w/ string

* Add decoding to python-experimental and regenerate

* Import re into python-experimental api_client

* Ensure file upload json response is utf-8 encoded bytes
---
 docs/generators/python-experimental.md        |  1 +
 docs/generators/python.md                     |  1 +
 .../languages/PythonClientCodegen.java        |  3 +-
 .../PythonClientExperimentalCodegen.java      |  4 +-
 .../main/resources/python/api_client.mustache | 47 ++++++++++-----
 .../resources/python/asyncio/rest.mustache    |  2 +-
 .../python-experimental/api_client.mustache   | 57 +++++++++++++------
 .../src/main/resources/python/rest.mustache   |  5 --
 .../resources/python/tornado/rest.mustache    |  7 +--
 .../python-asyncio/petstore_api/api_client.py | 42 +++++++++-----
 .../python-asyncio/petstore_api/rest.py       |  2 +-
 .../petstore_api/api_client.py                | 52 +++++++++++------
 .../python-experimental/petstore_api/rest.py  |  5 --
 .../test/test_format_test.py                  |  2 +-
 .../tests/test_api_exception.py               |  1 +
 .../tests/test_deserialization.py             | 30 ++++++++++
 .../python-experimental/tests/test_pet_api.py |  2 +-
 .../python-tornado/petstore_api/api_client.py | 42 +++++++++-----
 .../python-tornado/petstore_api/rest.py       |  7 +--
 .../python/petstore_api/api_client.py         | 42 +++++++++-----
 .../petstore/python/petstore_api/rest.py      |  5 --
 .../petstore/python/test/test_format_test.py  | 26 ++++-----
 .../python/tests/test_api_exception.py        |  1 +
 .../petstore_api/api_client.py                | 52 +++++++++++------
 .../python-experimental/petstore_api/rest.py  |  5 --
 .../python/petstore_api/api_client.py         | 42 +++++++++-----
 .../petstore/python/petstore_api/rest.py      |  5 --
 .../php-slim4/lib/Model/EnumClass.php         |  4 +-
 28 files changed, 319 insertions(+), 175 deletions(-)

diff --git a/docs/generators/python-experimental.md b/docs/generators/python-experimental.md
index 8f3b02c7e56..546dcc22542 100644
--- a/docs/generators/python-experimental.md
+++ b/docs/generators/python-experimental.md
@@ -31,6 +31,7 @@ sidebar_label: python-experimental
 
 <ul class="column-ul">
 <li>bool</li>
+<li>bytes</li>
 <li>date</li>
 <li>datetime</li>
 <li>dict</li>
diff --git a/docs/generators/python.md b/docs/generators/python.md
index 3bd96b865bc..7c4ff80016a 100644
--- a/docs/generators/python.md
+++ b/docs/generators/python.md
@@ -31,6 +31,7 @@ sidebar_label: python
 
 <ul class="column-ul">
 <li>bool</li>
+<li>bytes</li>
 <li>date</li>
 <li>datetime</li>
 <li>dict</li>
diff --git a/modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java b/modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java
index ca59f763d74..077676af99b 100644
--- a/modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java
+++ b/modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java
@@ -119,6 +119,7 @@ public class PythonClientCodegen extends DefaultCodegen implements CodegenConfig
         languageSpecificPrimitives.add("object");
         // TODO file and binary is mapped as `file`
         languageSpecificPrimitives.add("file");
+        languageSpecificPrimitives.add("bytes");
 
         typeMapping.clear();
         typeMapping.put("integer", "int");
@@ -828,7 +829,7 @@ public class PythonClientCodegen extends DefaultCodegen implements CodegenConfig
             if (schema.getDiscriminator()!=null) {
                 toExclude = schema.getDiscriminator().getPropertyName();
             }
-            
+
             example = packageName + ".models." + underscore(schema.getTitle())+"."+schema.getTitle()+"(";
 
             // if required only:
diff --git a/modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientExperimentalCodegen.java b/modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientExperimentalCodegen.java
index 6aa9c08250c..60a004b6bea 100644
--- a/modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientExperimentalCodegen.java
+++ b/modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientExperimentalCodegen.java
@@ -884,8 +884,8 @@ public class PythonClientExperimentalCodegen extends PythonClientCodegen {
      * Return a string representation of the Python types for the specified schema.
      * Primitive types in the OAS specification are implemented in Python using the corresponding
      * Python primitive types.
-     * Composed types (e.g. allAll, oneOf, anyOf) are represented in Python using list of types. 
-     * 
+     * Composed types (e.g. allAll, oneOf, anyOf) are represented in Python using list of types.
+     *
      * @param p The OAS schema.
      * @param prefix prepended to the returned value.
      * @param suffix appended to the returned value.
diff --git a/modules/openapi-generator/src/main/resources/python/api_client.mustache b/modules/openapi-generator/src/main/resources/python/api_client.mustache
index de25b7cf975..66d3577f3c2 100644
--- a/modules/openapi-generator/src/main/resources/python/api_client.mustache
+++ b/modules/openapi-generator/src/main/resources/python/api_client.mustache
@@ -22,7 +22,7 @@ import tornado.gen
 from {{packageName}}.configuration import Configuration
 import {{modelPackage}}
 from {{packageName}} import rest
-from {{packageName}}.exceptions import ApiValueError
+from {{packageName}}.exceptions import ApiValueError, ApiException
 
 
 class ApiClient(object):
@@ -186,22 +186,43 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = {{#asyncio}}await {{/asyncio}}{{#tornado}}yield {{/tornado}}self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = {{#asyncio}}await {{/asyncio}}{{#tornado}}yield {{/tornado}}self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(response_data, response_type)
-            else:
-                return_data = None
+
+        if not _preload_content:
+            {{^tornado}}
+            return return_data
+            {{/tornado}}
+            {{#tornado}}
+            raise tornado.gen.Return(return_data)
+            {{/tornado}}
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(response_data, response_type)
+        else:
+            return_data = None
 
 {{^tornado}}
         if _return_http_data_only:
diff --git a/modules/openapi-generator/src/main/resources/python/asyncio/rest.mustache b/modules/openapi-generator/src/main/resources/python/asyncio/rest.mustache
index f2099c75312..44531fce5a2 100644
--- a/modules/openapi-generator/src/main/resources/python/asyncio/rest.mustache
+++ b/modules/openapi-generator/src/main/resources/python/asyncio/rest.mustache
@@ -166,7 +166,7 @@ class RESTClientObject(object):
         r = await self.pool_manager.request(**args)
         if _preload_content:
 
-            data = await r.text()
+            data = await r.read()
             r = RESTResponse(r, data)
 
             # log response body
diff --git a/modules/openapi-generator/src/main/resources/python/python-experimental/api_client.mustache b/modules/openapi-generator/src/main/resources/python/python-experimental/api_client.mustache
index 774dae89c61..5285a7b8064 100644
--- a/modules/openapi-generator/src/main/resources/python/python-experimental/api_client.mustache
+++ b/modules/openapi-generator/src/main/resources/python/python-experimental/api_client.mustache
@@ -7,6 +7,7 @@ import atexit
 import mimetypes
 from multiprocessing.pool import ThreadPool
 import os
+import re
 
 # python 2 and python 3 compatibility library
 import six
@@ -17,7 +18,7 @@ import tornado.gen
 
 from {{packageName}} import rest
 from {{packageName}}.configuration import Configuration
-from {{packageName}}.exceptions import ApiValueError
+from {{packageName}}.exceptions import ApiValueError, ApiException
 from {{packageName}}.model_utils import (
     ModelNormal,
     ModelSimple,
@@ -176,26 +177,48 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = {{#asyncio}}await {{/asyncio}}{{#tornado}}yield {{/tornado}}self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = {{#asyncio}}await {{/asyncio}}{{#tornado}}yield {{/tornado}}self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(
-                    response_data,
-                    response_type,
-                    _check_type
-                )
-            else:
-                return_data = None
+
+        if not _preload_content:
+            {{^tornado}}
+            return (return_data)
+            {{/tornado}}
+            {{#tornado}}
+            raise tornado.gen.Return(return_data)
+            {{/tornado}}
+            return return_data
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(
+                response_data,
+                response_type,
+                _check_type
+            )
+        else:
+            return_data = None
 
 {{^tornado}}
         if _return_http_data_only:
diff --git a/modules/openapi-generator/src/main/resources/python/rest.mustache b/modules/openapi-generator/src/main/resources/python/rest.mustache
index 772efe91b39..fe7abfdfc04 100644
--- a/modules/openapi-generator/src/main/resources/python/rest.mustache
+++ b/modules/openapi-generator/src/main/resources/python/rest.mustache
@@ -209,11 +209,6 @@ class RESTClientObject(object):
         if _preload_content:
             r = RESTResponse(r)
 
-            # In the python 3, the response.data is bytes.
-            # we need to decode it to string.
-            if six.PY3:
-                r.data = r.data.decode('utf8')
-
             # log response body
             logger.debug("response body: %s", r.data)
 
diff --git a/modules/openapi-generator/src/main/resources/python/tornado/rest.mustache b/modules/openapi-generator/src/main/resources/python/tornado/rest.mustache
index e7a760f3706..2679760ea5b 100644
--- a/modules/openapi-generator/src/main/resources/python/tornado/rest.mustache
+++ b/modules/openapi-generator/src/main/resources/python/tornado/rest.mustache
@@ -8,7 +8,6 @@ import logging
 import re
 
 # python 2 and python 3 compatibility library
-import six
 from six.moves.urllib.parse import urlencode
 import tornado
 import tornado.gen
@@ -28,11 +27,7 @@ class RESTResponse(io.IOBase):
         self.reason = resp.reason
 
         if resp.body:
-            # In Python 3, the response body is utf-8 encoded bytes.
-            if six.PY3:
-                self.data = resp.body.decode('utf-8')
-            else:
-                self.data = resp.body
+            self.data = resp.body
         else:
             self.data = None
 
diff --git a/samples/client/petstore/python-asyncio/petstore_api/api_client.py b/samples/client/petstore/python-asyncio/petstore_api/api_client.py
index 8aae783bfc3..7e0d28cd4c6 100644
--- a/samples/client/petstore/python-asyncio/petstore_api/api_client.py
+++ b/samples/client/petstore/python-asyncio/petstore_api/api_client.py
@@ -27,7 +27,7 @@ from six.moves.urllib.parse import quote
 from petstore_api.configuration import Configuration
 import petstore_api.models
 from petstore_api import rest
-from petstore_api.exceptions import ApiValueError
+from petstore_api.exceptions import ApiValueError, ApiException
 
 
 class ApiClient(object):
@@ -177,22 +177,38 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = await self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = await self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(response_data, response_type)
-            else:
-                return_data = None
+
+        if not _preload_content:
+            return return_data
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(response_data, response_type)
+        else:
+            return_data = None
 
         if _return_http_data_only:
             return (return_data)
diff --git a/samples/client/petstore/python-asyncio/petstore_api/rest.py b/samples/client/petstore/python-asyncio/petstore_api/rest.py
index 1394af356ba..9310b9dfdec 100644
--- a/samples/client/petstore/python-asyncio/petstore_api/rest.py
+++ b/samples/client/petstore/python-asyncio/petstore_api/rest.py
@@ -174,7 +174,7 @@ class RESTClientObject(object):
         r = await self.pool_manager.request(**args)
         if _preload_content:
 
-            data = await r.text()
+            data = await r.read()
             r = RESTResponse(r, data)
 
             # log response body
diff --git a/samples/client/petstore/python-experimental/petstore_api/api_client.py b/samples/client/petstore/python-experimental/petstore_api/api_client.py
index 2e9d94da16c..39833326818 100644
--- a/samples/client/petstore/python-experimental/petstore_api/api_client.py
+++ b/samples/client/petstore/python-experimental/petstore_api/api_client.py
@@ -15,6 +15,7 @@ import atexit
 import mimetypes
 from multiprocessing.pool import ThreadPool
 import os
+import re
 
 # python 2 and python 3 compatibility library
 import six
@@ -22,7 +23,7 @@ from six.moves.urllib.parse import quote
 
 from petstore_api import rest
 from petstore_api.configuration import Configuration
-from petstore_api.exceptions import ApiValueError
+from petstore_api.exceptions import ApiValueError, ApiException
 from petstore_api.model_utils import (
     ModelNormal,
     ModelSimple,
@@ -178,26 +179,43 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(
-                    response_data,
-                    response_type,
-                    _check_type
-                )
-            else:
-                return_data = None
+
+        if not _preload_content:
+            return (return_data)
+            return return_data
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(
+                response_data,
+                response_type,
+                _check_type
+            )
+        else:
+            return_data = None
 
         if _return_http_data_only:
             return (return_data)
diff --git a/samples/client/petstore/python-experimental/petstore_api/rest.py b/samples/client/petstore/python-experimental/petstore_api/rest.py
index 7ed815b8a18..973161ca7ae 100644
--- a/samples/client/petstore/python-experimental/petstore_api/rest.py
+++ b/samples/client/petstore/python-experimental/petstore_api/rest.py
@@ -217,11 +217,6 @@ class RESTClientObject(object):
         if _preload_content:
             r = RESTResponse(r)
 
-            # In the python 3, the response.data is bytes.
-            # we need to decode it to string.
-            if six.PY3:
-                r.data = r.data.decode('utf8')
-
             # log response body
             logger.debug("response body: %s", r.data)
 
diff --git a/samples/client/petstore/python-experimental/test/test_format_test.py b/samples/client/petstore/python-experimental/test/test_format_test.py
index ed926aa031e..e7a32e3253f 100644
--- a/samples/client/petstore/python-experimental/test/test_format_test.py
+++ b/samples/client/petstore/python-experimental/test/test_format_test.py
@@ -150,4 +150,4 @@ class TestFormatTest(unittest.TestCase):
 
 
 if __name__ == '__main__':
-    unittest.main()
\ No newline at end of file
+    unittest.main()
diff --git a/samples/client/petstore/python-experimental/tests/test_api_exception.py b/samples/client/petstore/python-experimental/tests/test_api_exception.py
index 026ba5bd79c..b26ef5252bb 100644
--- a/samples/client/petstore/python-experimental/tests/test_api_exception.py
+++ b/samples/client/petstore/python-experimental/tests/test_api_exception.py
@@ -10,6 +10,7 @@ $ nosetests -v
 """
 
 import os
+import six
 import sys
 import unittest
 
diff --git a/samples/client/petstore/python-experimental/tests/test_deserialization.py b/samples/client/petstore/python-experimental/tests/test_deserialization.py
index 8ebeb1751ce..83ecabe81d2 100644
--- a/samples/client/petstore/python-experimental/tests/test_deserialization.py
+++ b/samples/client/petstore/python-experimental/tests/test_deserialization.py
@@ -377,6 +377,36 @@ class DeserializationTests(unittest.TestCase):
         finally:
             os.unlink(file_path)
 
+    def test_deserialize_binary_to_str(self):
+        """Ensures that bytes deserialization works"""
+        response_types_mixed = (str,)
+
+        # sample from http://www.jtricks.com/download-text
+        HTTPResponse = namedtuple(
+            'urllib3_response_HTTPResponse',
+            ['status', 'reason', 'data', 'getheaders', 'getheader']
+        )
+        headers = {}
+        def get_headers():
+            return headers
+        def get_header(name, default=None):
+            return headers.get(name, default)
+        data = "str"
+
+        http_response = HTTPResponse(
+            status=200,
+            reason='OK',
+            data=json.dumps(data).encode("utf-8") if six.PY3 else json.dumps(data),
+            getheaders=get_headers,
+            getheader=get_header
+        )
+
+        mock_response = RESTResponse(http_response)
+
+        result = self.deserialize(mock_response, response_types_mixed, True)
+        self.assertEqual(isinstance(result, str), True)
+        self.assertEqual(result, data)
+
     def test_deserialize_string_boolean_map(self):
         """
         Ensures that string boolean (additional properties)
diff --git a/samples/client/petstore/python-experimental/tests/test_pet_api.py b/samples/client/petstore/python-experimental/tests/test_pet_api.py
index 919d2505423..ee7e53ecad8 100644
--- a/samples/client/petstore/python-experimental/tests/test_pet_api.py
+++ b/samples/client/petstore/python-experimental/tests/test_pet_api.py
@@ -329,7 +329,7 @@ class PetApiTests(unittest.TestCase):
         http_response = HTTPResponse(
             status=200,
             reason='OK',
-            data=json.dumps(api_respponse),
+            data=json.dumps(api_respponse).encode('utf-8'),
             getheaders=get_headers,
             getheader=get_header
         )
diff --git a/samples/client/petstore/python-tornado/petstore_api/api_client.py b/samples/client/petstore/python-tornado/petstore_api/api_client.py
index 328c1a70e10..3fc10d72ab5 100644
--- a/samples/client/petstore/python-tornado/petstore_api/api_client.py
+++ b/samples/client/petstore/python-tornado/petstore_api/api_client.py
@@ -28,7 +28,7 @@ import tornado.gen
 from petstore_api.configuration import Configuration
 import petstore_api.models
 from petstore_api import rest
-from petstore_api.exceptions import ApiValueError
+from petstore_api.exceptions import ApiValueError, ApiException
 
 
 class ApiClient(object):
@@ -178,22 +178,38 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = yield self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = yield self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(response_data, response_type)
-            else:
-                return_data = None
+
+        if not _preload_content:
+            raise tornado.gen.Return(return_data)
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(response_data, response_type)
+        else:
+            return_data = None
 
         if _return_http_data_only:
             raise tornado.gen.Return(return_data)
diff --git a/samples/client/petstore/python-tornado/petstore_api/rest.py b/samples/client/petstore/python-tornado/petstore_api/rest.py
index d5661febf4b..f3f274480a6 100644
--- a/samples/client/petstore/python-tornado/petstore_api/rest.py
+++ b/samples/client/petstore/python-tornado/petstore_api/rest.py
@@ -16,7 +16,6 @@ import logging
 import re
 
 # python 2 and python 3 compatibility library
-import six
 from six.moves.urllib.parse import urlencode
 import tornado
 import tornado.gen
@@ -36,11 +35,7 @@ class RESTResponse(io.IOBase):
         self.reason = resp.reason
 
         if resp.body:
-            # In Python 3, the response body is utf-8 encoded bytes.
-            if six.PY3:
-                self.data = resp.body.decode('utf-8')
-            else:
-                self.data = resp.body
+            self.data = resp.body
         else:
             self.data = None
 
diff --git a/samples/client/petstore/python/petstore_api/api_client.py b/samples/client/petstore/python/petstore_api/api_client.py
index a80084e04b0..7b1b0010a49 100644
--- a/samples/client/petstore/python/petstore_api/api_client.py
+++ b/samples/client/petstore/python/petstore_api/api_client.py
@@ -27,7 +27,7 @@ from six.moves.urllib.parse import quote
 from petstore_api.configuration import Configuration
 import petstore_api.models
 from petstore_api import rest
-from petstore_api.exceptions import ApiValueError
+from petstore_api.exceptions import ApiValueError, ApiException
 
 
 class ApiClient(object):
@@ -176,22 +176,38 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(response_data, response_type)
-            else:
-                return_data = None
+
+        if not _preload_content:
+            return return_data
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(response_data, response_type)
+        else:
+            return_data = None
 
         if _return_http_data_only:
             return (return_data)
diff --git a/samples/client/petstore/python/petstore_api/rest.py b/samples/client/petstore/python/petstore_api/rest.py
index 7ed815b8a18..973161ca7ae 100644
--- a/samples/client/petstore/python/petstore_api/rest.py
+++ b/samples/client/petstore/python/petstore_api/rest.py
@@ -217,11 +217,6 @@ class RESTClientObject(object):
         if _preload_content:
             r = RESTResponse(r)
 
-            # In the python 3, the response.data is bytes.
-            # we need to decode it to string.
-            if six.PY3:
-                r.data = r.data.decode('utf8')
-
             # log response body
             logger.debug("response body: %s", r.data)
 
diff --git a/samples/client/petstore/python/test/test_format_test.py b/samples/client/petstore/python/test/test_format_test.py
index 70255fccd1d..e64c47b385d 100644
--- a/samples/client/petstore/python/test/test_format_test.py
+++ b/samples/client/petstore/python/test/test_format_test.py
@@ -36,19 +36,19 @@ class TestFormatTest(unittest.TestCase):
         # model = petstore_api.models.format_test.FormatTest()  # noqa: E501
         if include_optional :
             return FormatTest(
-                integer = 1E+1, 
-                int32 = 2E+1, 
-                int64 = 56, 
-                number = 32.1, 
-                float = 54.3, 
-                double = 67.8, 
-                string = 'a', 
-                byte = 'YQ==', 
-                binary = bytes(b'blah'), 
-                date = datetime.datetime.strptime('1975-12-30', '%Y-%m-%d').date(), 
-                date_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), 
-                uuid = '72f98069-206d-4f12-9f12-3d1e525a8e84', 
-                password = '0123456789', 
+                integer = 1E+1,
+                int32 = 2E+1,
+                int64 = 56,
+                number = 32.1,
+                float = 54.3,
+                double = 67.8,
+                string = 'a',
+                byte = 'YQ==',
+                binary = 'bytes',
+                date = datetime.datetime.strptime('1975-12-30', '%Y-%m-%d').date(),
+                date_time = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'),
+                uuid = '72f98069-206d-4f12-9f12-3d1e525a8e84',
+                password = '0123456789',
                 big_decimal = 1
             )
         else :
diff --git a/samples/client/petstore/python/tests/test_api_exception.py b/samples/client/petstore/python/tests/test_api_exception.py
index 75bf81a8de0..b076628c0a0 100644
--- a/samples/client/petstore/python/tests/test_api_exception.py
+++ b/samples/client/petstore/python/tests/test_api_exception.py
@@ -10,6 +10,7 @@ $ nosetests -v
 """
 
 import os
+import six
 import sys
 import unittest
 
diff --git a/samples/openapi3/client/petstore/python-experimental/petstore_api/api_client.py b/samples/openapi3/client/petstore/python-experimental/petstore_api/api_client.py
index 544a21c79dd..1edccecef67 100644
--- a/samples/openapi3/client/petstore/python-experimental/petstore_api/api_client.py
+++ b/samples/openapi3/client/petstore/python-experimental/petstore_api/api_client.py
@@ -15,6 +15,7 @@ import atexit
 import mimetypes
 from multiprocessing.pool import ThreadPool
 import os
+import re
 
 # python 2 and python 3 compatibility library
 import six
@@ -22,7 +23,7 @@ from six.moves.urllib.parse import quote
 
 from petstore_api import rest
 from petstore_api.configuration import Configuration
-from petstore_api.exceptions import ApiValueError
+from petstore_api.exceptions import ApiValueError, ApiException
 from petstore_api.model_utils import (
     ModelNormal,
     ModelSimple,
@@ -178,26 +179,43 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(
-                    response_data,
-                    response_type,
-                    _check_type
-                )
-            else:
-                return_data = None
+
+        if not _preload_content:
+            return (return_data)
+            return return_data
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(
+                response_data,
+                response_type,
+                _check_type
+            )
+        else:
+            return_data = None
 
         if _return_http_data_only:
             return (return_data)
diff --git a/samples/openapi3/client/petstore/python-experimental/petstore_api/rest.py b/samples/openapi3/client/petstore/python-experimental/petstore_api/rest.py
index 7ed815b8a18..973161ca7ae 100644
--- a/samples/openapi3/client/petstore/python-experimental/petstore_api/rest.py
+++ b/samples/openapi3/client/petstore/python-experimental/petstore_api/rest.py
@@ -217,11 +217,6 @@ class RESTClientObject(object):
         if _preload_content:
             r = RESTResponse(r)
 
-            # In the python 3, the response.data is bytes.
-            # we need to decode it to string.
-            if six.PY3:
-                r.data = r.data.decode('utf8')
-
             # log response body
             logger.debug("response body: %s", r.data)
 
diff --git a/samples/openapi3/client/petstore/python/petstore_api/api_client.py b/samples/openapi3/client/petstore/python/petstore_api/api_client.py
index a80084e04b0..7b1b0010a49 100644
--- a/samples/openapi3/client/petstore/python/petstore_api/api_client.py
+++ b/samples/openapi3/client/petstore/python/petstore_api/api_client.py
@@ -27,7 +27,7 @@ from six.moves.urllib.parse import quote
 from petstore_api.configuration import Configuration
 import petstore_api.models
 from petstore_api import rest
-from petstore_api.exceptions import ApiValueError
+from petstore_api.exceptions import ApiValueError, ApiException
 
 
 class ApiClient(object):
@@ -176,22 +176,38 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(response_data, response_type)
-            else:
-                return_data = None
+
+        if not _preload_content:
+            return return_data
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(response_data, response_type)
+        else:
+            return_data = None
 
         if _return_http_data_only:
             return (return_data)
diff --git a/samples/openapi3/client/petstore/python/petstore_api/rest.py b/samples/openapi3/client/petstore/python/petstore_api/rest.py
index 7ed815b8a18..973161ca7ae 100644
--- a/samples/openapi3/client/petstore/python/petstore_api/rest.py
+++ b/samples/openapi3/client/petstore/python/petstore_api/rest.py
@@ -217,11 +217,6 @@ class RESTClientObject(object):
         if _preload_content:
             r = RESTResponse(r)
 
-            # In the python 3, the response.data is bytes.
-            # we need to decode it to string.
-            if six.PY3:
-                r.data = r.data.decode('utf8')
-
             # log response body
             logger.debug("response body: %s", r.data)
 
diff --git a/samples/server/petstore/php-slim4/lib/Model/EnumClass.php b/samples/server/petstore/php-slim4/lib/Model/EnumClass.php
index 3ddd829b313..c3998d8602c 100644
--- a/samples/server/petstore/php-slim4/lib/Model/EnumClass.php
+++ b/samples/server/petstore/php-slim4/lib/Model/EnumClass.php
@@ -30,8 +30,8 @@ class EnumClass implements ModelInterface
     private const MODEL_SCHEMA = <<<'SCHEMA'
 {
   "type" : "string",
-  "default" : "-efg",
-  "enum" : [ "_abc", "-efg", "(xyz)" ]
+  "enum" : [ "_abc", "-efg", "(xyz)" ],
+  "default" : "-efg"
 }
 SCHEMA;