fix: Configure python urllib3 connection pool size (#17323)

This was removed in #16802, but using a higher value than 1,
or at least making this configurable makes complete sense.

Without this, we get a lot of these log messages:

[ WARNING] Connection pool is full, discarding connection:
This commit is contained in:
Robert Schweizer
2023-12-06 08:49:10 +01:00
committed by GitHub
parent 6230248716
commit 6a43a371f1
8 changed files with 48 additions and 0 deletions

View File

@@ -15,6 +15,7 @@
import copy
import logging
import multiprocessing
import sys
import urllib3
@@ -164,6 +165,13 @@ conf = openapi_client.Configuration(
Set this to the SNI value expected by the server.
"""
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5
"""urllib3 connection pool's maximum number of connections saved
per pool. urllib3 uses 1 connection as default value, but this is
not the best value when you are making a lot of possibly parallel
requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance.
"""
self.proxy = None
"""Proxy URL

View File

@@ -88,6 +88,9 @@ class RESTClientObject:
if configuration.socket_options is not None:
addition_pool_args['socket_options'] = configuration.socket_options
if configuration.connection_pool_maxsize is not None:
addition_pool_args['maxsize'] = configuration.connection_pool_maxsize
# https pool manager
if configuration.proxy:
if is_socks_proxy_url(configuration.proxy):