feat: configurable limit of simultaneous connections (python/asyncio) (#3200)

* feat: configurable limit of simultaneous connections (python/asyncio)

* fix: remove unused import (python/asyncio)
This commit is contained in:
Tomasz Prus
2019-06-25 08:13:43 +02:00
committed by William Cheng
parent f681764067
commit 5b9283beca
4 changed files with 21 additions and 9 deletions

View File

@@ -38,8 +38,11 @@ class RESTResponse(io.IOBase):
class RESTClientObject(object): class RESTClientObject(object):
def __init__(self, configuration, pools_size=4, maxsize=4): def __init__(self, configuration, pools_size=4, maxsize=None):
# maxsize is number of requests to host that are allowed in parallel # maxsize is number of requests to host that are allowed in parallel
if maxsize is None:
maxsize = configuration.connection_pool_maxsize
# ca_certs # ca_certs
if configuration.ssl_ca_cert: if configuration.ssl_ca_cert:

View File

@@ -6,7 +6,9 @@ from __future__ import absolute_import
import copy import copy
import logging import logging
{{^asyncio}}
import multiprocessing import multiprocessing
{{/asyncio}}
import sys import sys
import urllib3 import urllib3
@@ -116,6 +118,13 @@ class Configuration(six.with_metaclass(TypeWithDefault, object)):
"""Set this to True/False to enable/disable SSL hostname verification. """Set this to True/False to enable/disable SSL hostname verification.
""" """
{{#asyncio}}
self.connection_pool_maxsize = 100
"""This value is passed to the aiohttp to limit simultaneous connections.
Default values is 100, None means no-limit.
"""
{{/asyncio}}
{{^asyncio}}
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5 self.connection_pool_maxsize = multiprocessing.cpu_count() * 5
"""urllib3 connection pool's maximum number of connections saved """urllib3 connection pool's maximum number of connections saved
per pool. urllib3 uses 1 connection as default value, but this is per pool. urllib3 uses 1 connection as default value, but this is
@@ -123,6 +132,7 @@ class Configuration(six.with_metaclass(TypeWithDefault, object)):
requests to the same host, which is often the case here. requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance. cpu_count * 5 is used as default value to increase performance.
""" """
{{/asyncio}}
self.proxy = None self.proxy = None
"""Proxy URL """Proxy URL

View File

@@ -14,7 +14,6 @@ from __future__ import absolute_import
import copy import copy
import logging import logging
import multiprocessing
import sys import sys
import urllib3 import urllib3
@@ -115,12 +114,9 @@ class Configuration(six.with_metaclass(TypeWithDefault, object)):
"""Set this to True/False to enable/disable SSL hostname verification. """Set this to True/False to enable/disable SSL hostname verification.
""" """
self.connection_pool_maxsize = multiprocessing.cpu_count() * 5 self.connection_pool_maxsize = 100
"""urllib3 connection pool's maximum number of connections saved """This value is passed to the aiohttp to limit simultaneous connections.
per pool. urllib3 uses 1 connection as default value, but this is Default values is 100, None means no-limit.
not the best value when you are making a lot of possibly parallel
requests to the same host, which is often the case here.
cpu_count * 5 is used as default value to increase performance.
""" """
self.proxy = None self.proxy = None

View File

@@ -46,8 +46,11 @@ class RESTResponse(io.IOBase):
class RESTClientObject(object): class RESTClientObject(object):
def __init__(self, configuration, pools_size=4, maxsize=4): def __init__(self, configuration, pools_size=4, maxsize=None):
# maxsize is number of requests to host that are allowed in parallel # maxsize is number of requests to host that are allowed in parallel
if maxsize is None:
maxsize = configuration.connection_pool_maxsize
# ca_certs # ca_certs
if configuration.ssl_ca_cert: if configuration.ssl_ca_cert: