feat: configurable limit of simultaneous connections (python/asyncio) (#3200)

* feat: configurable limit of simultaneous connections (python/asyncio)

* fix: remove unused import (python/asyncio)
This commit is contained in:
Tomasz Prus
2019-06-25 08:13:43 +02:00
committed by William Cheng
parent f681764067
commit 5b9283beca
4 changed files with 21 additions and 9 deletions

View File

@@ -14,7 +14,6 @@ from __future__ import absolute_import
import copy
import logging
import multiprocessing
import sys
import urllib3
@@ -115,12 +114,9 @@ class Configuration(six.with_metaclass(TypeWithDefault, object)):
"""Set this to True/False to enable/disable SSL hostname verification.
"""
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5
"""urllib3 connection pool's maximum number of connections saved
per pool. urllib3 uses 1 connection as default value, but this is
not the best value when you are making a lot of possibly parallel
requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance.
self.connection_pool_maxsize = 100
"""This value is passed to the aiohttp to limit simultaneous connections.
Default values is 100, None means no-limit.
"""
self.proxy = None

View File

@@ -46,8 +46,11 @@ class RESTResponse(io.IOBase):
class RESTClientObject(object):
def __init__(self, configuration, pools_size=4, maxsize=4):
def __init__(self, configuration, pools_size=4, maxsize=None):
# maxsize is number of requests to host that are allowed in parallel
if maxsize is None:
maxsize = configuration.connection_pool_maxsize
# ca_certs
if configuration.ssl_ca_cert: