[Python] Fix Python UTF-8 Encoding Issue (#5679)

* Try decoding but don't bail on error

* Switch binary and ByteArray to bytes

* Read content type and parse appropriately

* Remove response parsing

* Remove response parsing and just return the data

* Update petshop examples w/ new generator code

* Fix copy/paste error with naming

* Update petstore examples

* Move response decoding to inside _preload_content block

* Update the clients again

* Use a raw string for the regex pattern

* Regenerate petstore clients

* Add bytes to python primitives as it's supported in 2.7 and 3

* Add bytes to the exports from model_utils

* Import bytes from model_utils

* Add conditional typing for regex pattern to match variable type

* Regenerate petstore clients

* Use read() instead of text() for asyncio

* Regenerate petstore clients

* Remove unused six import

* Regenerate petstore clients

* Add newline to kick Circle to re-run

* Remove whitespace from tox.ini

* Update more examples after ensure_updated

* Add sample updates that didn't run with the --batch flag

* Remove extra bracket in regex to remove warning

* Stop printing debug messages

* Add bytes examples to python doc generators

* Update generated FakeApi docs

* Regenerate api_client.py

* Remove print statements from generated clients

* Update bytes example in FakeApi.md. Again. I swear.

* Add yet another seemingly missing doc update

* Catch the error, decode the body, and re-throw

* Remove the updates now that the change is non-breaking

* Regenerate client

* Add bytes deserialization test

* Update exception parsing

* Add exception parsing for python-experimental

* Regenerate client with minor changes

* Revert test changes

* Regenerate model_utils.py

* Update confusing test name

* Remove bytes from mapping and examples

* Add back in the old binary/ByteArray to str mapping

* Update docs and api_client template

* Add experimental api_client changes

* Regenerate samples again

* Add Tornado handling to early return

* Try fixing Tornado python returns

* More documentation changes

* Re-generate the client code

* Remove bytes from test_format_test

* Remove more leftover bytes usages

* Switch bytes validation back to string

* Fix format_test template and regenerate

* Remove unused bytes var

* Remove bytes import from models and regenerate

* Remove bytes import from test_deserialization

* Reduce nested ifs

* Remove byte logic for now

* Regenerate client after latest changes

* Remove another bytes usage

* Regenerate after removing dangling byte string usage

* Reduce the scope of the try/catch in api_client

* Regenerate after try/catch cleanup

* Swap catch for except

* Regenerate Python client after api_client change

* Fix lint error on the generated api_client

* Add binary format test back in w/ string

* Add decoding to python-experimental and regenerate

* Import re into python-experimental api_client

* Ensure file upload json response is utf-8 encoded bytes

samples/client/petstore/python-asyncio/petstore_api/api_client.py

@@ -27,7 +27,7 @@ from six.moves.urllib.parse import quote
 from petstore_api.configuration import Configuration
 import petstore_api.models
 from petstore_api import rest
-from petstore_api.exceptions import ApiValueError
+from petstore_api.exceptions import ApiValueError, ApiException
 
 
 class ApiClient(object):
@@ -177,22 +177,38 @@ class ApiClient(object):
             # use server/host defined in path or operation instead
             url = _host + resource_path
 
-        # perform request and return response
-        response_data = await self.request(
-            method, url, query_params=query_params, headers=header_params,
-            post_params=post_params, body=body,
-            _preload_content=_preload_content,
-            _request_timeout=_request_timeout)
+        try:
+            # perform request and return response
+            response_data = await self.request(
+                method, url, query_params=query_params, headers=header_params,
+                post_params=post_params, body=body,
+                _preload_content=_preload_content,
+                _request_timeout=_request_timeout)
+        except ApiException as e:
+            e.body = e.body.decode('utf-8') if six.PY3 else e.body
+            raise e
+
+        content_type = response_data.getheader('content-type')
 
         self.last_response = response_data
 
         return_data = response_data
-        if _preload_content:
-            # deserialize response data
-            if response_type:
-                return_data = self.deserialize(response_data, response_type)
-            else:
-                return_data = None
+
+        if not _preload_content:
+            return return_data
+
+        if six.PY3 and response_type not in ["file", "bytes"]:
+            match = None
+            if content_type is not None:
+                match = re.search(r"charset=([a-zA-Z\-\d]+)[\s\;]?", content_type)
+            encoding = match.group(1) if match else "utf-8"
+            response_data.data = response_data.data.decode(encoding)
+
+        # deserialize response data
+        if response_type:
+            return_data = self.deserialize(response_data, response_type)
+        else:
+            return_data = None
 
         if _return_http_data_only:
             return (return_data)

samples/client/petstore/python-asyncio/petstore_api/rest.py

@@ -174,7 +174,7 @@ class RESTClientObject(object):
         r = await self.pool_manager.request(**args)
         if _preload_content:
 
-            data = await r.text()
+            data = await r.read()
             r = RESTResponse(r, data)
 
             # log response body