[python-experimental] fixes json + charset use case (#12114)

* Adds code to detect json content type when charset is also set

* Updates template to properly render content type, regenerates samples

* Adds test_json_with_charset

* Reverts version file

* Fixes typo

modules/openapi-generator/src/main/resources/python-experimental/api_client.handlebars

@@ -719,7 +719,20 @@ class ApiResponseWithoutDeserialization(ApiResponse):
     headers: typing.Union[Unset, typing.List[HeaderParameter]] = unset
 
 
-class OpenApiResponse:
+class JSONDetector:
+    @staticmethod
+    def content_type_is_json(content_type: str) -> bool:
+        """
+        for when content_type strings also include charset info like:
+        application/json; charset=UTF-8
+        """
+        content_type_piece = content_type.split(';')[0]
+        if content_type_piece == 'application/json':
+            return True
+        return False
+
+
+class OpenApiResponse(JSONDetector):
     def __init__(
         self,
         response_cls: typing.Type[ApiResponse] = ApiResponse,
@@ -734,8 +747,8 @@ class OpenApiResponse:
 
     @staticmethod
     def __deserialize_json(response: urllib3.HTTPResponse) -> typing.Any:
-        decoded_data = response.data.decode("utf-8")
-        return json.loads(decoded_data)
+        # python must be >= 3.9 so we can pass in bytes into json.loads
+        return json.loads(response.data)
 
     @staticmethod
     def __file_name_from_content_disposition(content_disposition: typing.Optional[str]) -> typing.Optional[str]:
@@ -796,7 +809,7 @@ class OpenApiResponse:
         deserialized_body = unset
         streamed = response.supports_chunked_reads()
         if self.content is not None:
-            if content_type == 'application/json':
+            if self.content_type_is_json(content_type):
                 body_data = self.__deserialize_json(response)
             elif content_type == 'application/octet-stream':
                 body_data = self.__deserialize_application_octet_stream(response)
@@ -1245,7 +1258,7 @@ class SerializedRequestBody(typing.TypedDict, total=False):
     fields: typing.Tuple[typing.Union[RequestField, tuple[str, str]], ...]
 
 
-class RequestBody(StyleFormSerializer):
+class RequestBody(StyleFormSerializer, JSONDetector):
     """
     A request body parameter
     content: content_type to MediaType Schema info
@@ -1382,7 +1395,7 @@ class RequestBody(StyleFormSerializer):
             cast_in_data = media_type.schema(in_data)
         # TODO check for and use encoding if it exists
         # and content_type is multipart or application/x-www-form-urlencoded
-        if content_type == 'application/json':
+        if self.content_type_is_json(content_type):
             return self.__serialize_json(cast_in_data)
         elif content_type == 'text/plain':
             return self.__serialize_text_plain(cast_in_data)

modules/openapi-generator/src/main/resources/python-experimental/endpoint.handlebars

@@ -182,7 +182,7 @@ class RequestCookieParams(RequestRequiredCookieParams, RequestOptionalCookiePara
 request_body_{{paramName}} = api_client.RequestBody(
     content={
 {{#each content}}
-        '{{@key}}': api_client.MediaType(
+        '{{{@key}}}': api_client.MediaType(
             schema={{this.schema.baseName}}),
 {{/each}}
     },
@@ -323,7 +323,7 @@ _response_for_{{code}} = api_client.OpenApiResponse(
 {{#if @first}}
     content={
 {{/if}}
-        '{{@key}}': api_client.MediaType(
+        '{{{@key}}}': api_client.MediaType(
             schema={{this.schema.baseName}}),
 {{#if @last}}
     },
@@ -351,7 +351,7 @@ _status_code_to_response = {
 {{#if @first}}
 _all_accept_content_types = (
 {{/if}}
-    '{{this.mediaType}}',
+    '{{{this.mediaType}}}',
 {{#if @last}}
 )
 {{/if}}
@@ -382,7 +382,7 @@ class {{operationIdCamelCase}}(api_client.Api):
     {{#with bodyParam}}
     {{#each content}}
     {{#if @first}}
-        content_type: str = '{{@key}}',
+        content_type: str = '{{{@key}}}',
     {{/if}}
     {{/each}}
     {{/with}}

modules/openapi-generator/src/test/resources/3_0/python-experimental/petstore-with-fake-endpoints-models-for-testing-with-http-signature.yaml

@@ -1530,6 +1530,22 @@ paths:
       responses:
         '200':
           description: ok
+  '/fake/jsonWithCharset':
+    post:
+      tags:
+        - fake
+      summary: json with charset tx and rx
+      operationId: jsonWithCharset
+      requestBody:
+        content:
+          application/json; charset=utf-8:
+            schema: {}
+      responses:
+        200:
+          description: success
+          content:
+            application/json; charset=utf-8:
+              schema: {}
 servers:
   - url: 'http://{server}.swagger.io:{port}/v2'
     description: petstore server

samples/openapi3/client/petstore/python-experimental/README.md

@@ -100,6 +100,7 @@ Class | Method | HTTP request | Description
 *FakeApi* | [**inline_additional_properties**](docs/FakeApi.md#inline_additional_properties) | **POST** /fake/inline-additionalProperties | test inline additionalProperties
 *FakeApi* | [**inline_composition**](docs/FakeApi.md#inline_composition) | **POST** /fake/inlineComposition/ | testing composed schemas at inline locations
 *FakeApi* | [**json_form_data**](docs/FakeApi.md#json_form_data) | **GET** /fake/jsonFormData | test json serialization of form data
+*FakeApi* | [**json_with_charset**](docs/FakeApi.md#json_with_charset) | **POST** /fake/jsonWithCharset | json with charset tx and rx
 *FakeApi* | [**mammal**](docs/FakeApi.md#mammal) | **POST** /fake/refs/mammal | 
 *FakeApi* | [**number_with_validations**](docs/FakeApi.md#number_with_validations) | **POST** /fake/refs/number | 
 *FakeApi* | [**object_in_query**](docs/FakeApi.md#object_in_query) | **GET** /fake/objInQuery | user list

samples/openapi3/client/petstore/python-experimental/docs/FakeApi.md

@@ -20,6 +20,7 @@ Method | HTTP request | Description
 [**inline_additional_properties**](FakeApi.md#inline_additional_properties) | **POST** /fake/inline-additionalProperties | test inline additionalProperties
 [**inline_composition**](FakeApi.md#inline_composition) | **POST** /fake/inlineComposition/ | testing composed schemas at inline locations
 [**json_form_data**](FakeApi.md#json_form_data) | **GET** /fake/jsonFormData | test json serialization of form data
+[**json_with_charset**](FakeApi.md#json_with_charset) | **POST** /fake/jsonWithCharset | json with charset tx and rx
 [**mammal**](FakeApi.md#mammal) | **POST** /fake/refs/mammal | 
 [**number_with_validations**](FakeApi.md#number_with_validations) | **POST** /fake/refs/number | 
 [**object_in_query**](FakeApi.md#object_in_query) | **GET** /fake/objInQuery | user list
@@ -1618,6 +1619,87 @@ No authorization required
 
 [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
 
+# **json_with_charset**
+> bool, date, datetime, dict, float, int, list, str, none_type json_with_charset()
+
+json with charset tx and rx
+
+### Example
+
+```python
+import petstore_api
+from petstore_api.api import fake_api
+from pprint import pprint
+# Defining the host is optional and defaults to http://petstore.swagger.io:80/v2
+# See configuration.py for a list of all supported configuration parameters.
+configuration = petstore_api.Configuration(
+    host = "http://petstore.swagger.io:80/v2"
+)
+
+# Enter a context with an instance of the API client
+with petstore_api.ApiClient(configuration) as api_client:
+    # Create an instance of the API class
+    api_instance = fake_api.FakeApi(api_client)
+
+    # example passing only optional values
+    body = None
+    try:
+        # json with charset tx and rx
+        api_response = api_instance.json_with_charset(
+            body=body,
+        )
+        pprint(api_response)
+    except petstore_api.ApiException as e:
+        print("Exception when calling FakeApi->json_with_charset: %s\n" % e)
+```
+### Parameters
+
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+body | typing.Union[SchemaForRequestBodyApplicationJsonCharsetutf8, Unset] | optional, default is unset |
+content_type | str | optional, default is 'application/json; charset=utf-8' | Selects the schema and serialization of the request body
+accept_content_types | typing.Tuple[str] | default is ('application/json; charset=utf-8', ) | Tells the server the content type(s) that are accepted by the client
+stream | bool | default is False | if True then the response.content will be streamed and loaded from a file like object. When downloading a file, set this to True to force the code to deserialize the content to a FileSchema file
+timeout | typing.Optional[typing.Union[int, typing.Tuple]] | default is None | the timeout used by the rest client
+skip_deserialization | bool | default is False | when True, headers and body will be unset and an instance of api_client.ApiResponseWithoutDeserialization will be returned
+
+### body
+
+#### SchemaForRequestBodyApplicationJsonCharsetutf8
+
+Type | Description | Notes
+------------- | ------------- | -------------
+typing.Union[dict, frozendict, str, date, datetime, int, float, bool, Decimal, None, list, tuple, bytes] | |
+
+### Return Types, Responses
+
+Code | Class | Description
+------------- | ------------- | -------------
+n/a | api_client.ApiResponseWithoutDeserialization | When skip_deserialization is True this response is returned
+200 | ApiResponseFor200 | success
+
+#### ApiResponseFor200
+Name | Type | Description  | Notes
+------------- | ------------- | ------------- | -------------
+response | urllib3.HTTPResponse | Raw response |
+body | typing.Union[SchemaFor200ResponseBodyApplicationJsonCharsetutf8, ] |  |
+headers | Unset | headers were not defined |
+
+#### SchemaFor200ResponseBodyApplicationJsonCharsetutf8
+
+Type | Description | Notes
+------------- | ------------- | -------------
+typing.Union[dict, frozendict, str, date, datetime, int, float, bool, Decimal, None, list, tuple, bytes] | |
+
+
+**bool, date, datetime, dict, float, int, list, str, none_type**
+
+### Authorization
+
+No authorization required
+
+[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
+
 # **mammal**
 > Mammal mammal(mammal)
 

samples/openapi3/client/petstore/python-experimental/petstore_api/api/fake_api.py

@@ -26,6 +26,7 @@ from petstore_api.api.fake_api_endpoints.group_parameters import GroupParameters
 from petstore_api.api.fake_api_endpoints.inline_additional_properties import InlineAdditionalProperties
 from petstore_api.api.fake_api_endpoints.inline_composition import InlineComposition
 from petstore_api.api.fake_api_endpoints.json_form_data import JsonFormData
+from petstore_api.api.fake_api_endpoints.json_with_charset import JsonWithCharset
 from petstore_api.api.fake_api_endpoints.mammal import Mammal
 from petstore_api.api.fake_api_endpoints.number_with_validations import NumberWithValidations
 from petstore_api.api.fake_api_endpoints.object_in_query import ObjectInQuery
@@ -57,6 +58,7 @@ class FakeApi(
     InlineAdditionalProperties,
     InlineComposition,
     JsonFormData,
+    JsonWithCharset,
     Mammal,
     NumberWithValidations,
     ObjectInQuery,

samples/openapi3/client/petstore/python-experimental/petstore_api/api/fake_api_endpoints/json_with_charset.py

@@ -0,0 +1,161 @@
+# coding: utf-8
+
+"""
+
+
+    Generated by: https://openapi-generator.tech
+"""
+
+from dataclasses import dataclass
+import re  # noqa: F401
+import sys  # noqa: F401
+import typing
+import urllib3
+from urllib3._collections import HTTPHeaderDict
+
+from petstore_api import api_client, exceptions
+import decimal  # noqa: F401
+from datetime import date, datetime  # noqa: F401
+from frozendict import frozendict  # noqa: F401
+
+from petstore_api.schemas import (  # noqa: F401
+    AnyTypeSchema,
+    ComposedSchema,
+    DictSchema,
+    ListSchema,
+    StrSchema,
+    IntSchema,
+    Int32Schema,
+    Int64Schema,
+    Float32Schema,
+    Float64Schema,
+    NumberSchema,
+    DateSchema,
+    DateTimeSchema,
+    DecimalSchema,
+    BoolSchema,
+    BinarySchema,
+    NoneSchema,
+    none_type,
+    Configuration,
+    Unset,
+    unset,
+    ComposedBase,
+    ListBase,
+    DictBase,
+    NoneBase,
+    StrBase,
+    IntBase,
+    Int32Base,
+    Int64Base,
+    Float32Base,
+    Float64Base,
+    NumberBase,
+    DateBase,
+    DateTimeBase,
+    BoolBase,
+    BinaryBase,
+    Schema,
+    _SchemaValidator,
+    _SchemaTypeChecker,
+    _SchemaEnumMaker
+)
+
+# body param
+SchemaForRequestBodyApplicationJsonCharsetutf8 = AnyTypeSchema
+
+
+request_body_body = api_client.RequestBody(
+    content={
+        'application/json; charset=utf-8': api_client.MediaType(
+            schema=SchemaForRequestBodyApplicationJsonCharsetutf8),
+    },
+)
+_path = '/fake/jsonWithCharset'
+_method = 'POST'
+SchemaFor200ResponseBodyApplicationJsonCharsetutf8 = AnyTypeSchema
+
+
+@dataclass
+class ApiResponseFor200(api_client.ApiResponse):
+    response: urllib3.HTTPResponse
+    body: typing.Union[
+        SchemaFor200ResponseBodyApplicationJsonCharsetutf8,
+    ]
+    headers: Unset = unset
+
+
+_response_for_200 = api_client.OpenApiResponse(
+    response_cls=ApiResponseFor200,
+    content={
+        'application/json; charset=utf-8': api_client.MediaType(
+            schema=SchemaFor200ResponseBodyApplicationJsonCharsetutf8),
+    },
+)
+_status_code_to_response = {
+    '200': _response_for_200,
+}
+_all_accept_content_types = (
+    'application/json; charset=utf-8',
+)
+
+
+class JsonWithCharset(api_client.Api):
+
+    def json_with_charset(
+        self: api_client.Api,
+        body: typing.Union[SchemaForRequestBodyApplicationJsonCharsetutf8, Unset] = unset,
+        content_type: str = 'application/json; charset=utf-8',
+        accept_content_types: typing.Tuple[str] = _all_accept_content_types,
+        stream: bool = False,
+        timeout: typing.Optional[typing.Union[int, typing.Tuple]] = None,
+        skip_deserialization: bool = False,
+    ) -> typing.Union[
+        ApiResponseFor200,
+        api_client.ApiResponseWithoutDeserialization
+    ]:
+        """
+        json with charset tx and rx
+        :param skip_deserialization: If true then api_response.response will be set but
+            api_response.body and api_response.headers will not be deserialized into schema
+            class instances
+        """
+
+        _headers = HTTPHeaderDict()
+        # TODO add cookie handling
+        if accept_content_types:
+            for accept_content_type in accept_content_types:
+                _headers.add('Accept', accept_content_type)
+
+        _fields = None
+        _body = None
+        if body is not unset:
+            serialized_data = request_body_body.serialize(body, content_type)
+            _headers.add('Content-Type', content_type)
+            if 'fields' in serialized_data:
+                _fields = serialized_data['fields']
+            elif 'body' in serialized_data:
+                _body = serialized_data['body']
+        response = self.api_client.call_api(
+            resource_path=_path,
+            method=_method,
+            headers=_headers,
+            fields=_fields,
+            body=_body,
+            stream=stream,
+            timeout=timeout,
+        )
+
+        if skip_deserialization:
+            api_response = api_client.ApiResponseWithoutDeserialization(response=response)
+        else:
+            response_for_status = _status_code_to_response.get(str(response.status))
+            if response_for_status:
+                api_response = response_for_status.deserialize(response, self.api_client.configuration)
+            else:
+                api_response = api_client.ApiResponseWithoutDeserialization(response=response)
+
+        if not 200 <= response.status <= 299:
+            raise exceptions.ApiException(api_response=api_response)
+
+        return api_response

samples/openapi3/client/petstore/python-experimental/petstore_api/api_client.py

@@ -723,7 +723,20 @@ class ApiResponseWithoutDeserialization(ApiResponse):
     headers: typing.Union[Unset, typing.List[HeaderParameter]] = unset
 
 
-class OpenApiResponse:
+class JSONDetector:
+    @staticmethod
+    def content_type_is_json(content_type: str) -> bool:
+        """
+        for when content_type strings also include charset info like:
+        application/json; charset=UTF-8
+        """
+        content_type_piece = content_type.split(';')[0]
+        if content_type_piece == 'application/json':
+            return True
+        return False
+
+
+class OpenApiResponse(JSONDetector):
     def __init__(
         self,
         response_cls: typing.Type[ApiResponse] = ApiResponse,
@@ -738,8 +751,8 @@ class OpenApiResponse:
 
     @staticmethod
     def __deserialize_json(response: urllib3.HTTPResponse) -> typing.Any:
-        decoded_data = response.data.decode("utf-8")
-        return json.loads(decoded_data)
+        # python must be >= 3.9 so we can pass in bytes into json.loads
+        return json.loads(response.data)
 
     @staticmethod
     def __file_name_from_content_disposition(content_disposition: typing.Optional[str]) -> typing.Optional[str]:
@@ -800,7 +813,7 @@ class OpenApiResponse:
         deserialized_body = unset
         streamed = response.supports_chunked_reads()
         if self.content is not None:
-            if content_type == 'application/json':
+            if self.content_type_is_json(content_type):
                 body_data = self.__deserialize_json(response)
             elif content_type == 'application/octet-stream':
                 body_data = self.__deserialize_application_octet_stream(response)
@@ -1244,7 +1257,7 @@ class SerializedRequestBody(typing.TypedDict, total=False):
     fields: typing.Tuple[typing.Union[RequestField, tuple[str, str]], ...]
 
 
-class RequestBody(StyleFormSerializer):
+class RequestBody(StyleFormSerializer, JSONDetector):
     """
     A request body parameter
     content: content_type to MediaType Schema info
@@ -1381,7 +1394,7 @@ class RequestBody(StyleFormSerializer):
             cast_in_data = media_type.schema(in_data)
         # TODO check for and use encoding if it exists
         # and content_type is multipart or application/x-www-form-urlencoded
-        if content_type == 'application/json':
+        if self.content_type_is_json(content_type):
             return self.__serialize_json(cast_in_data)
         elif content_type == 'text/plain':
             return self.__serialize_text_plain(cast_in_data)

samples/openapi3/client/petstore/python-experimental/tests_manual/test_fake_api.py

@@ -679,6 +679,29 @@ class TestFakeApi(unittest.TestCase):
                     accept_content_types=(content_type,)
                 )
 
+    def test_json_with_charset(self):
+        # serialization + deserialization of json with charset works
+        with patch.object(RESTClientObject, 'request') as mock_request:
+            body = None
+            content_type_with_charset = 'application/json; charset=utf-8'
+            mock_request.return_value = self.__response(
+                self.__json_bytes(body),
+                content_type=content_type_with_charset
+            )
+
+            api_response = self.api.json_with_charset(body=body)
+            self.__assert_request_called_with(
+                mock_request,
+                'http://petstore.swagger.io:80/v2/fake/jsonWithCharset',
+                body=self.__json_bytes(body),
+                content_type=content_type_with_charset,
+                accept_content_type=content_type_with_charset
+            )
+
+            assert isinstance(api_response.body, schemas.AnyTypeSchema)
+            assert isinstance(api_response.body, schemas.NoneClass)
+            assert api_response.body.is_none()
+
 
 if __name__ == '__main__':
     unittest.main()