bf1eebb · searx · git.iko.soy

Merge pull request #549 from guyou/feature-pool-fine-tuning

Use new settings to fine tune the connection pool

commitbf1eebb17f2a4e4454d59a59757e0265e33c9dcf authorAdam Tauber <asciimoo@gmail.com> dateThu 19 May 2016, 15:51 (10 years ago) parent9247b1e d571a9d

2 files changed+10−4

unifiedsplit

searx/poolrequests.py+8−4

@@ -41,14 +41,18 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):

4141 block=self._pool_block, **self._conn_params)

4242

4343

44+connect = settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code

45+maxsize = settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor

4446if settings['outgoing'].get('source_ips'):

45− http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))

47+ http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize,

48+ source_address=(source_ip, 0))

4649 for source_ip in settings['outgoing']['source_ips'])

47− https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))

50+ https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize,

51+ source_address=(source_ip, 0))

4852 for source_ip in settings['outgoing']['source_ips'])

4953else:

50− http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))

51− https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))

54+ http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), ))

55+ https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), ))

5256

5357

5458class SessionSinglePool(requests.Session):

searx/settings.yml+2

@@ -21,6 +21,8 @@ ui:

2121outgoing: # communication with search engines

2222 request_timeout : 2.0 # seconds

2323 useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator

24+ pool_connections : 100 # Number of different hosts

25+ pool_maxsize : 10 # Number of simultaneous requests by host

2426# uncomment below section if you want to use a proxy

2527# see http://docs.python-requests.org/en/latest/user/advanced/#proxies

2628# SOCKS proxies are not supported : see https://github.com/kennethreitz/requests/pull/478