From d39b1ff76faad5fd3575c1b4ee0c95c7e5dd0303 Mon Sep 17 00:00:00 2001 From: Yusuke Tsutsumi Date: Mon, 14 Aug 2017 01:16:55 -0700 Subject: [PATCH] [python] asyncio support (#6245) * trying an approach with providing asyncio as a framework. * adding example of asyncio. * removing sample client to help PR look more manageable. * possibly fixing a unit test * getting unit test to pass again. * addressing comments. --- .gitignore | 2 +- bin/python-asyncio-petstore.sh | 31 +++ .../languages/PythonClientCodegen.java | 22 +- .../main/resources/python/api_client.mustache | 4 +- .../resources/python/asyncio/rest.mustache | 242 ++++++++++++++++++ .../src/main/resources/python/setup.mustache | 3 + .../options/PythonClientOptionsProvider.java | 1 + .../python/PythonClientOptionsTest.java | 4 +- 8 files changed, 300 insertions(+), 9 deletions(-) create mode 100755 bin/python-asyncio-petstore.sh create mode 100644 modules/swagger-codegen/src/main/resources/python/asyncio/rest.mustache diff --git a/.gitignore b/.gitignore index 0b4554fd56b..291d302ff08 100644 --- a/.gitignore +++ b/.gitignore @@ -166,6 +166,6 @@ samples/client/petstore/typescript-angular/tsd-debug.log # aspnetcore samples/server/petstore/aspnetcore/.vs/ effective.pom - # kotlin samples/client/petstore/kotlin/src/main/kotlin/test/ +\? diff --git a/bin/python-asyncio-petstore.sh b/bin/python-asyncio-petstore.sh new file mode 100755 index 00000000000..83f23b0ee9b --- /dev/null +++ b/bin/python-asyncio-petstore.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +SCRIPT="$0" + +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +if [ ! -d "${APP_DIR}" ]; then + APP_DIR=`dirname "$SCRIPT"`/.. + APP_DIR=`cd "${APP_DIR}"; pwd` +fi + +executable="./modules/swagger-codegen-cli/target/swagger-codegen-cli.jar" + +if [ ! -f "$executable" ] +then + mvn clean package +fi + +# if you've executed sbt assembly previously it will use that instead. +export JAVA_OPTS="${JAVA_OPTS} -XX:MaxPermSize=256M -Xmx1024M -DloggerPath=conf/log4j.properties" +ags="generate -t modules/swagger-codegen/src/main/resources/python -i modules/swagger-codegen/src/test/resources/2_0/petstore-with-fake-endpoints-models-for-testing.yaml -l python -o samples/client/petstore/python-asyncio -DpackageName=petstore_api --library asyncio $@" + +java $JAVA_OPTS -jar $executable $ags diff --git a/modules/swagger-codegen/src/main/java/io/swagger/codegen/languages/PythonClientCodegen.java b/modules/swagger-codegen/src/main/java/io/swagger/codegen/languages/PythonClientCodegen.java index 21e76d9f265..e82e4dbc2d4 100755 --- a/modules/swagger-codegen/src/main/java/io/swagger/codegen/languages/PythonClientCodegen.java +++ b/modules/swagger-codegen/src/main/java/io/swagger/codegen/languages/PythonClientCodegen.java @@ -21,8 +21,11 @@ import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + public class PythonClientCodegen extends DefaultCodegen implements CodegenConfig { public static final String PACKAGE_URL = "packageUrl"; + public static final String DEFAULT_LIBRARY = "urllib3"; protected String packageName; // e.g. petstore_api protected String packageVersion; @@ -124,6 +127,13 @@ public class PythonClientCodegen extends DefaultCodegen implements CodegenConfig CodegenConstants.SORT_PARAMS_BY_REQUIRED_FLAG_DESC).defaultValue(Boolean.TRUE.toString())); cliOptions.add(new CliOption(CodegenConstants.HIDE_GENERATION_TIMESTAMP, "hides the timestamp when files were generated") .defaultValue(Boolean.TRUE.toString())); + + supportedLibraries.put("urllib3", "urllib3-based client"); + supportedLibraries.put("asyncio", "Asyncio-based client (python 3.5+)"); + CliOption libraryOption = new CliOption(CodegenConstants.LIBRARY, "library template (sub-template) to use"); + libraryOption.setDefault(DEFAULT_LIBRARY); + cliOptions.add(libraryOption); + setLibrary(DEFAULT_LIBRARY); } @Override @@ -185,13 +195,10 @@ public class PythonClientCodegen extends DefaultCodegen implements CodegenConfig supportingFiles.add(new SupportingFile("README.mustache", "", "README.md")); - supportingFiles.add(new SupportingFile("setup.mustache", "", "setup.py")); supportingFiles.add(new SupportingFile("tox.mustache", "", "tox.ini")); supportingFiles.add(new SupportingFile("test-requirements.mustache", "", "test-requirements.txt")); supportingFiles.add(new SupportingFile("requirements.mustache", "", "requirements.txt")); - supportingFiles.add(new SupportingFile("api_client.mustache", swaggerFolder, "api_client.py")); - supportingFiles.add(new SupportingFile("rest.mustache", swaggerFolder, "rest.py")); supportingFiles.add(new SupportingFile("configuration.mustache", swaggerFolder, "configuration.py")); supportingFiles.add(new SupportingFile("__init__package.mustache", swaggerFolder, "__init__.py")); supportingFiles.add(new SupportingFile("__init__model.mustache", modelPackage, "__init__.py")); @@ -203,6 +210,15 @@ public class PythonClientCodegen extends DefaultCodegen implements CodegenConfig supportingFiles.add(new SupportingFile("git_push.sh.mustache", "", "git_push.sh")); supportingFiles.add(new SupportingFile("gitignore.mustache", "", ".gitignore")); supportingFiles.add(new SupportingFile("travis.mustache", "", ".travis.yml")); + supportingFiles.add(new SupportingFile("setup.mustache", "", "setup.py")); + supportingFiles.add(new SupportingFile("api_client.mustache", swaggerFolder, "api_client.py")); + + if ("asyncio".equals(getLibrary())) { + supportingFiles.add(new SupportingFile("asyncio/rest.mustache", swaggerFolder, "rest.py")); + additionalProperties.put("asyncio", "true"); + } else { + supportingFiles.add(new SupportingFile("rest.mustache", swaggerFolder, "rest.py")); + } } private static String dropDots(String str) { diff --git a/modules/swagger-codegen/src/main/resources/python/api_client.mustache b/modules/swagger-codegen/src/main/resources/python/api_client.mustache index 1a6ecf25db3..ceba9229227 100644 --- a/modules/swagger-codegen/src/main/resources/python/api_client.mustache +++ b/modules/swagger-codegen/src/main/resources/python/api_client.mustache @@ -81,7 +81,7 @@ class ApiClient(object): def set_default_header(self, header_name, header_value): self.default_headers[header_name] = header_value - def __call_api(self, resource_path, method, + {{#asyncio}}async {{/asyncio}}def __call_api(self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, @@ -134,7 +134,7 @@ class ApiClient(object): url = self.configuration.host + resource_path # perform request and return response - response_data = self.request(method, url, + response_data = {{#asyncio}}await {{/asyncio}}self.request(method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, diff --git a/modules/swagger-codegen/src/main/resources/python/asyncio/rest.mustache b/modules/swagger-codegen/src/main/resources/python/asyncio/rest.mustache new file mode 100644 index 00000000000..8acfa9a2b89 --- /dev/null +++ b/modules/swagger-codegen/src/main/resources/python/asyncio/rest.mustache @@ -0,0 +1,242 @@ +# coding: utf-8 + +{{>partial_header}} + +import aiohttp +import io +import json +import ssl +import certifi +import logging +import re + +# python 2 and python 3 compatibility library +from six import PY3 +from six.moves.urllib.parse import urlencode + +logger = logging.getLogger(__name__) + + +class RESTResponse(io.IOBase): + + def __init__(self, resp, data): + self.aiohttp_response = resp + self.status = resp.status + self.reason = resp.reason + self.data = data + + def getheaders(self): + """ + Returns a CIMultiDictProxy of the response headers. + """ + return self.aiohttp_response.headers + + def getheader(self, name, default=None): + """ + Returns a given response header. + """ + return self.aiohttp_response.headers.get(name, default) + + +class RESTClientObject: + + def __init__(self, configuration, pools_size=4, maxsize=4): + # maxsize is the number of requests to host that are allowed in parallel + # ca_certs vs cert_file vs key_file + # http://stackoverflow.com/a/23957365/2985775 + + # ca_certs + if configuration.ssl_ca_cert: + ca_certs = configuration.ssl_ca_cert + else: + # if not set certificate file, use Mozilla's root certificates. + ca_certs = certifi.where() + + ssl_context = ssl.SSLContext() + if configuration.cert_file: + ssl_context.load_cert_chain( + configuration.cert_file, keyfile=configuration.key_file + ) + + connector = aiohttp.TCPConnector( + limit=maxsize, + verify_ssl=configuration.verify_ssl + ) + + # https pool manager + if configuration.proxy: + self.pool_manager = aiohttp.ClientSession( + connector=connector, + proxy=configuration.proxy + ) + else: + self.pool_manager = aiohttp.ClientSession( + connector=connector + ) + + async def request(self, method, url, query_params=None, headers=None, + body=None, post_params=None, _preload_content=True, _request_timeout=None): + """ + :param method: http request method + :param url: http request url + :param query_params: query parameters in the url + :param headers: http request headers + :param body: request json body, for `application/json` + :param post_params: request post parameters, + `application/x-www-form-urlencoded` + and `multipart/form-data` + :param _preload_content: this is a non-applicable field for the AiohttpClient. + :param _request_timeout: timeout setting for this request. If one number provided, it will be total request + timeout. It can also be a pair (tuple) of (connection, read) timeouts. + """ + method = method.upper() + assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS'] + + if post_params and body: + raise ValueError( + "body parameter cannot be used with post_params parameter." + ) + + post_params = post_params or {} + headers = headers or {} + timeout = _request_timeout or 5 * 60 + + if 'Content-Type' not in headers: + headers['Content-Type'] = 'application/json' + + args = { + "method": method, + "url": url, + "timeout": timeout, + "headers": headers + } + # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` + if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: + if query_params: + url += '?' + urlencode(query_params) + if re.search('json', headers['Content-Type'], re.IGNORECASE): + if body: + body = json.dumps(body) + args["data"] = body + elif headers['Content-Type'] == 'application/x-www-form-urlencoded': + data = aiohttp.FormData() + for k, v in post_params.items(): + data.add_field(k, v) + args["data"] = data + elif headers['Content-Type'] == 'multipart/form-data': + args["data"] = post_params + # Pass a `bytes` parameter directly in the body to support + # other content types than Json when `body` argument is provided + # in serialized form + elif isinstance(body, bytes): + args["data"] = body + else: + # Cannot generate the request from given parameters + msg = """Cannot prepare a request message for provided arguments. + Please check that your arguments match declared content type.""" + raise ApiException(status=0, reason=msg) + else: + args["data"] = query_params + + async with self.pool_manager.request(**args) as r: + data = await r.text() + r = RESTResponse(r, data) + + # log response body + logger.debug("response body: %s", r.data) + + if not 200 <= r.status <= 299: + raise ApiException(http_resp=r) + + return r + + async def GET(self, url, headers=None, query_params=None, _preload_content=True, _request_timeout=None): + return (await self.request("GET", url, + headers=headers, + _preload_content=_preload_content, + _request_timeout=_request_timeout, + query_params=query_params)) + + async def HEAD(self, url, headers=None, query_params=None, _preload_content=True, _request_timeout=None): + return (await self.request("HEAD", url, + headers=headers, + _preload_content=_preload_content, + _request_timeout=_request_timeout, + query_params=query_params)) + + async def OPTIONS(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, + _request_timeout=None): + return (await self.request("OPTIONS", url, + headers=headers, + query_params=query_params, + post_params=post_params, + _preload_content=_preload_content, + _request_timeout=_request_timeout, + body=body)) + + async def DELETE(self, url, headers=None, query_params=None, body=None, _preload_content=True, _request_timeout=None): + return (await self.request("DELETE", url, + headers=headers, + query_params=query_params, + _preload_content=_preload_content, + _request_timeout=_request_timeout, + body=body)) + + async def POST(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, + _request_timeout=None): + return (await self.request("POST", url, + headers=headers, + query_params=query_params, + post_params=post_params, + _preload_content=_preload_content, + _request_timeout=_request_timeout, + body=body)) + + async def PUT(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, + _request_timeout=None): + return (await self.request("PUT", url, + headers=headers, + query_params=query_params, + post_params=post_params, + _preload_content=_preload_content, + _request_timeout=_request_timeout, + body=body)) + + async def PATCH(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, + _request_timeout=None): + return (await self.request("PATCH", url, + headers=headers, + query_params=query_params, + post_params=post_params, + _preload_content=_preload_content, + _request_timeout=_request_timeout, + body=body)) + + +class ApiException(Exception): + + def __init__(self, status=None, reason=None, http_resp=None): + if http_resp: + self.status = http_resp.status + self.reason = http_resp.reason + self.body = http_resp.data + self.headers = http_resp.getheaders() + else: + self.status = status + self.reason = reason + self.body = None + self.headers = None + + def __str__(self): + """ + Custom error messages for exception + """ + error_message = "({0})\n"\ + "Reason: {1}\n".format(self.status, self.reason) + if self.headers: + error_message += "HTTP response headers: {0}\n".format(self.headers) + + if self.body: + error_message += "HTTP response body: {0}\n".format(self.body) + + return error_message diff --git a/modules/swagger-codegen/src/main/resources/python/setup.mustache b/modules/swagger-codegen/src/main/resources/python/setup.mustache index 6fc7f198d15..d3fd92e52ac 100644 --- a/modules/swagger-codegen/src/main/resources/python/setup.mustache +++ b/modules/swagger-codegen/src/main/resources/python/setup.mustache @@ -18,6 +18,9 @@ VERSION = "{{packageVersion}}" # http://pypi.python.org/pypi/setuptools REQUIRES = ["urllib3 >= 1.15", "six >= 1.10", "certifi", "python-dateutil"] +{{#asyncio}} +REQUIRES.append("aiohttp") +{{/asyncio}} setup( name=NAME, diff --git a/modules/swagger-codegen/src/test/java/io/swagger/codegen/options/PythonClientOptionsProvider.java b/modules/swagger-codegen/src/test/java/io/swagger/codegen/options/PythonClientOptionsProvider.java index 262c312e563..1eccdd764e1 100644 --- a/modules/swagger-codegen/src/test/java/io/swagger/codegen/options/PythonClientOptionsProvider.java +++ b/modules/swagger-codegen/src/test/java/io/swagger/codegen/options/PythonClientOptionsProvider.java @@ -27,6 +27,7 @@ public class PythonClientOptionsProvider implements OptionsProvider { .put(CodegenConstants.PACKAGE_VERSION, PACKAGE_VERSION_VALUE) .put(CodegenConstants.SORT_PARAMS_BY_REQUIRED_FLAG, "true") .put(CodegenConstants.HIDE_GENERATION_TIMESTAMP, "true") + .put(CodegenConstants.LIBRARY, "urllib3") .build(); } diff --git a/modules/swagger-codegen/src/test/java/io/swagger/codegen/python/PythonClientOptionsTest.java b/modules/swagger-codegen/src/test/java/io/swagger/codegen/python/PythonClientOptionsTest.java index 8529ea0c97d..0ded3781592 100644 --- a/modules/swagger-codegen/src/test/java/io/swagger/codegen/python/PythonClientOptionsTest.java +++ b/modules/swagger-codegen/src/test/java/io/swagger/codegen/python/PythonClientOptionsTest.java @@ -27,12 +27,10 @@ public class PythonClientOptionsTest extends AbstractOptionsTest { protected void setExpectations() { new Expectations(clientCodegen) {{ clientCodegen.setPackageName(PythonClientOptionsProvider.PACKAGE_NAME_VALUE); - times = 1; clientCodegen.setProjectName(PythonClientOptionsProvider.PROJECT_NAME_VALUE); - times = 1; clientCodegen.setPackageVersion(PythonClientOptionsProvider.PACKAGE_VERSION_VALUE); - times = 1; clientCodegen.setPackageUrl(PythonClientOptionsProvider.PACKAGE_URL_VALUE); + // clientCodegen.setLibrary(PythonClientCodegen.DEFAULT_LIBRARY); times = 1; }}; }