git.iko.soy/searx
on master help sign in
Merge pull request #549 from guyou/feature-pool-fine-tuning
bf1eebbAdam Tauber
searx/poolrequests.py+8−4
compare with master
unifiedsplit
@@ -41,14 +41,18 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):
41 block=self._pool_block, **self._conn_params)
41 block=self._pool_block, **self._conn_params)
42
42
43
43
44+connect = settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code
45+maxsize = settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor
44if settings['outgoing'].get('source_ips'):
46if settings['outgoing'].get('source_ips'):
45− http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
47+ http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize,
48+ source_address=(source_ip, 0))
46 for source_ip in settings['outgoing']['source_ips'])
49 for source_ip in settings['outgoing']['source_ips'])
47− https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
50+ https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize,
51+ source_address=(source_ip, 0))
48 for source_ip in settings['outgoing']['source_ips'])
52 for source_ip in settings['outgoing']['source_ips'])
49else:
53else:
50− http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
54+ http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), ))
51− https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
55+ https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=connect, pool_maxsize=maxsize), ))
52
56
53
57
54class SessionSinglePool(requests.Session):
58class SessionSinglePool(requests.Session):
searx/settings.yml+2
@@ -21,6 +21,8 @@ ui:
21outgoing: # communication with search engines
21outgoing: # communication with search engines
22 request_timeout : 2.0 # seconds
22 request_timeout : 2.0 # seconds
23 useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
23 useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
24+ pool_connections : 100 # Number of different hosts
25+ pool_maxsize : 10 # Number of simultaneous requests by host
24# uncomment below section if you want to use a proxy
26# uncomment below section if you want to use a proxy
25# see http://docs.python-requests.org/en/latest/user/advanced/#proxies
27# see http://docs.python-requests.org/en/latest/user/advanced/#proxies
26# SOCKS proxies are not supported : see https://github.com/kennethreitz/requests/pull/478
28# SOCKS proxies are not supported : see https://github.com/kennethreitz/requests/pull/478