fix: Configure python urllib3 connection pool size (#17323)

This was removed in #16802, but using a higher value than 1,
or at least making this configurable makes complete sense.

Without this, we get a lot of these log messages:

[ WARNING] Connection pool is full, discarding connection:
This commit is contained in:
Robert Schweizer 2023-12-06 08:49:10 +01:00 committed by GitHub
parent 6230248716
commit 6a43a371f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 48 additions and 0 deletions

View File

@ -4,6 +4,9 @@
import copy import copy
import logging import logging
{{^asyncio}}
import multiprocessing
{{/asyncio}}
import sys import sys
import urllib3 import urllib3
@ -239,6 +242,15 @@ conf = {{{packageName}}}.Configuration(
Default values is 100, None means no-limit. Default values is 100, None means no-limit.
""" """
{{/asyncio}} {{/asyncio}}
{{^asyncio}}
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5
"""urllib3 connection pool's maximum number of connections saved
per pool. urllib3 uses 1 connection as default value, but this is
not the best value when you are making a lot of possibly parallel
requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance.
"""
{{/asyncio}}
self.proxy = None self.proxy = None
"""Proxy URL """Proxy URL

View File

@ -77,6 +77,9 @@ class RESTClientObject:
if configuration.socket_options is not None: if configuration.socket_options is not None:
addition_pool_args['socket_options'] = configuration.socket_options addition_pool_args['socket_options'] = configuration.socket_options
if configuration.connection_pool_maxsize is not None:
addition_pool_args['maxsize'] = configuration.connection_pool_maxsize
# https pool manager # https pool manager
if configuration.proxy: if configuration.proxy:
if is_socks_proxy_url(configuration.proxy): if is_socks_proxy_url(configuration.proxy):

View File

@ -15,6 +15,7 @@
import copy import copy
import logging import logging
import multiprocessing
import sys import sys
import urllib3 import urllib3
@ -164,6 +165,13 @@ conf = openapi_client.Configuration(
Set this to the SNI value expected by the server. Set this to the SNI value expected by the server.
""" """
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5
"""urllib3 connection pool's maximum number of connections saved
per pool. urllib3 uses 1 connection as default value, but this is
not the best value when you are making a lot of possibly parallel
requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance.
"""
self.proxy = None self.proxy = None
"""Proxy URL """Proxy URL

View File

@ -88,6 +88,9 @@ class RESTClientObject:
if configuration.socket_options is not None: if configuration.socket_options is not None:
addition_pool_args['socket_options'] = configuration.socket_options addition_pool_args['socket_options'] = configuration.socket_options
if configuration.connection_pool_maxsize is not None:
addition_pool_args['maxsize'] = configuration.connection_pool_maxsize
# https pool manager # https pool manager
if configuration.proxy: if configuration.proxy:
if is_socks_proxy_url(configuration.proxy): if is_socks_proxy_url(configuration.proxy):

View File

@ -15,6 +15,7 @@
import copy import copy
import logging import logging
import multiprocessing
import sys import sys
import urllib3 import urllib3
@ -164,6 +165,13 @@ conf = openapi_client.Configuration(
Set this to the SNI value expected by the server. Set this to the SNI value expected by the server.
""" """
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5
"""urllib3 connection pool's maximum number of connections saved
per pool. urllib3 uses 1 connection as default value, but this is
not the best value when you are making a lot of possibly parallel
requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance.
"""
self.proxy = None self.proxy = None
"""Proxy URL """Proxy URL

View File

@ -88,6 +88,9 @@ class RESTClientObject:
if configuration.socket_options is not None: if configuration.socket_options is not None:
addition_pool_args['socket_options'] = configuration.socket_options addition_pool_args['socket_options'] = configuration.socket_options
if configuration.connection_pool_maxsize is not None:
addition_pool_args['maxsize'] = configuration.connection_pool_maxsize
# https pool manager # https pool manager
if configuration.proxy: if configuration.proxy:
if is_socks_proxy_url(configuration.proxy): if is_socks_proxy_url(configuration.proxy):

View File

@ -14,6 +14,7 @@
import copy import copy
import logging import logging
import multiprocessing
import sys import sys
import urllib3 import urllib3
@ -229,6 +230,13 @@ conf = petstore_api.Configuration(
Set this to the SNI value expected by the server. Set this to the SNI value expected by the server.
""" """
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5
"""urllib3 connection pool's maximum number of connections saved
per pool. urllib3 uses 1 connection as default value, but this is
not the best value when you are making a lot of possibly parallel
requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance.
"""
self.proxy = None self.proxy = None
"""Proxy URL """Proxy URL

View File

@ -87,6 +87,9 @@ class RESTClientObject:
if configuration.socket_options is not None: if configuration.socket_options is not None:
addition_pool_args['socket_options'] = configuration.socket_options addition_pool_args['socket_options'] = configuration.socket_options
if configuration.connection_pool_maxsize is not None:
addition_pool_args['maxsize'] = configuration.connection_pool_maxsize
# https pool manager # https pool manager
if configuration.proxy: if configuration.proxy:
if is_socks_proxy_url(configuration.proxy): if is_socks_proxy_url(configuration.proxy):