From 6a43a371f19628b631f0cebb9c9f2a55509a99c5 Mon Sep 17 00:00:00 2001 From: Robert Schweizer Date: Wed, 6 Dec 2023 08:49:10 +0100 Subject: [PATCH] fix: Configure python urllib3 connection pool size (#17323) This was removed in #16802, but using a higher value than 1, or at least making this configurable makes complete sense. Without this, we get a lot of these log messages: [ WARNING] Connection pool is full, discarding connection: --- .../src/main/resources/python/configuration.mustache | 12 ++++++++++++ .../src/main/resources/python/rest.mustache | 3 +++ .../openapi_client/configuration.py | 8 ++++++++ .../openapi_client/rest.py | 3 +++ .../echo_api/python/openapi_client/configuration.py | 8 ++++++++ .../client/echo_api/python/openapi_client/rest.py | 3 +++ .../petstore/python/petstore_api/configuration.py | 8 ++++++++ .../client/petstore/python/petstore_api/rest.py | 3 +++ 8 files changed, 48 insertions(+) diff --git a/modules/openapi-generator/src/main/resources/python/configuration.mustache b/modules/openapi-generator/src/main/resources/python/configuration.mustache index a5c74424d80..c3799c0c5e0 100644 --- a/modules/openapi-generator/src/main/resources/python/configuration.mustache +++ b/modules/openapi-generator/src/main/resources/python/configuration.mustache @@ -4,6 +4,9 @@ import copy import logging +{{^asyncio}} +import multiprocessing +{{/asyncio}} import sys import urllib3 @@ -239,6 +242,15 @@ conf = {{{packageName}}}.Configuration( Default values is 100, None means no-limit. """ {{/asyncio}} + {{^asyncio}} + self.connection_pool_maxsize = multiprocessing.cpu_count() * 5 + """urllib3 connection pool's maximum number of connections saved + per pool. urllib3 uses 1 connection as default value, but this is + not the best value when you are making a lot of possibly parallel + requests to the same host, which is often the case here. + cpu_count * 5 is used as default value to increase performance. + """ + {{/asyncio}} self.proxy = None """Proxy URL diff --git a/modules/openapi-generator/src/main/resources/python/rest.mustache b/modules/openapi-generator/src/main/resources/python/rest.mustache index 1fc57bd83e9..f0998845859 100644 --- a/modules/openapi-generator/src/main/resources/python/rest.mustache +++ b/modules/openapi-generator/src/main/resources/python/rest.mustache @@ -77,6 +77,9 @@ class RESTClientObject: if configuration.socket_options is not None: addition_pool_args['socket_options'] = configuration.socket_options + if configuration.connection_pool_maxsize is not None: + addition_pool_args['maxsize'] = configuration.connection_pool_maxsize + # https pool manager if configuration.proxy: if is_socks_proxy_url(configuration.proxy): diff --git a/samples/client/echo_api/python-disallowAdditionalPropertiesIfNotPresent-true/openapi_client/configuration.py b/samples/client/echo_api/python-disallowAdditionalPropertiesIfNotPresent-true/openapi_client/configuration.py index b7218cb4ff2..7c6f3e11f99 100644 --- a/samples/client/echo_api/python-disallowAdditionalPropertiesIfNotPresent-true/openapi_client/configuration.py +++ b/samples/client/echo_api/python-disallowAdditionalPropertiesIfNotPresent-true/openapi_client/configuration.py @@ -15,6 +15,7 @@ import copy import logging +import multiprocessing import sys import urllib3 @@ -164,6 +165,13 @@ conf = openapi_client.Configuration( Set this to the SNI value expected by the server. """ + self.connection_pool_maxsize = multiprocessing.cpu_count() * 5 + """urllib3 connection pool's maximum number of connections saved + per pool. urllib3 uses 1 connection as default value, but this is + not the best value when you are making a lot of possibly parallel + requests to the same host, which is often the case here. + cpu_count * 5 is used as default value to increase performance. + """ self.proxy = None """Proxy URL diff --git a/samples/client/echo_api/python-disallowAdditionalPropertiesIfNotPresent-true/openapi_client/rest.py b/samples/client/echo_api/python-disallowAdditionalPropertiesIfNotPresent-true/openapi_client/rest.py index 86762867f67..5d0ca946c1f 100644 --- a/samples/client/echo_api/python-disallowAdditionalPropertiesIfNotPresent-true/openapi_client/rest.py +++ b/samples/client/echo_api/python-disallowAdditionalPropertiesIfNotPresent-true/openapi_client/rest.py @@ -88,6 +88,9 @@ class RESTClientObject: if configuration.socket_options is not None: addition_pool_args['socket_options'] = configuration.socket_options + if configuration.connection_pool_maxsize is not None: + addition_pool_args['maxsize'] = configuration.connection_pool_maxsize + # https pool manager if configuration.proxy: if is_socks_proxy_url(configuration.proxy): diff --git a/samples/client/echo_api/python/openapi_client/configuration.py b/samples/client/echo_api/python/openapi_client/configuration.py index b7218cb4ff2..7c6f3e11f99 100644 --- a/samples/client/echo_api/python/openapi_client/configuration.py +++ b/samples/client/echo_api/python/openapi_client/configuration.py @@ -15,6 +15,7 @@ import copy import logging +import multiprocessing import sys import urllib3 @@ -164,6 +165,13 @@ conf = openapi_client.Configuration( Set this to the SNI value expected by the server. """ + self.connection_pool_maxsize = multiprocessing.cpu_count() * 5 + """urllib3 connection pool's maximum number of connections saved + per pool. urllib3 uses 1 connection as default value, but this is + not the best value when you are making a lot of possibly parallel + requests to the same host, which is often the case here. + cpu_count * 5 is used as default value to increase performance. + """ self.proxy = None """Proxy URL diff --git a/samples/client/echo_api/python/openapi_client/rest.py b/samples/client/echo_api/python/openapi_client/rest.py index 86762867f67..5d0ca946c1f 100644 --- a/samples/client/echo_api/python/openapi_client/rest.py +++ b/samples/client/echo_api/python/openapi_client/rest.py @@ -88,6 +88,9 @@ class RESTClientObject: if configuration.socket_options is not None: addition_pool_args['socket_options'] = configuration.socket_options + if configuration.connection_pool_maxsize is not None: + addition_pool_args['maxsize'] = configuration.connection_pool_maxsize + # https pool manager if configuration.proxy: if is_socks_proxy_url(configuration.proxy): diff --git a/samples/openapi3/client/petstore/python/petstore_api/configuration.py b/samples/openapi3/client/petstore/python/petstore_api/configuration.py index 0d1672dcb41..8bca6dc7ab0 100755 --- a/samples/openapi3/client/petstore/python/petstore_api/configuration.py +++ b/samples/openapi3/client/petstore/python/petstore_api/configuration.py @@ -14,6 +14,7 @@ import copy import logging +import multiprocessing import sys import urllib3 @@ -229,6 +230,13 @@ conf = petstore_api.Configuration( Set this to the SNI value expected by the server. """ + self.connection_pool_maxsize = multiprocessing.cpu_count() * 5 + """urllib3 connection pool's maximum number of connections saved + per pool. urllib3 uses 1 connection as default value, but this is + not the best value when you are making a lot of possibly parallel + requests to the same host, which is often the case here. + cpu_count * 5 is used as default value to increase performance. + """ self.proxy = None """Proxy URL diff --git a/samples/openapi3/client/petstore/python/petstore_api/rest.py b/samples/openapi3/client/petstore/python/petstore_api/rest.py index a7ba897c3ed..ff75022d695 100755 --- a/samples/openapi3/client/petstore/python/petstore_api/rest.py +++ b/samples/openapi3/client/petstore/python/petstore_api/rest.py @@ -87,6 +87,9 @@ class RESTClientObject: if configuration.socket_options is not None: addition_pool_args['socket_options'] = configuration.socket_options + if configuration.connection_pool_maxsize is not None: + addition_pool_args['maxsize'] = configuration.connection_pool_maxsize + # https pool manager if configuration.proxy: if is_socks_proxy_url(configuration.proxy):