@@ -1,20 +1,63 @@
11import requests
2+from itertools import cycle
3+from searx import settings
24
35
4−the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
5−the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
6+class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):
7+
8+ def __init__(self, pool_connections=requests.adapters.DEFAULT_POOLSIZE,
9+ pool_maxsize=requests.adapters.DEFAULT_POOLSIZE,
10+ max_retries=requests.adapters.DEFAULT_RETRIES,
11+ pool_block=requests.adapters.DEFAULT_POOLBLOCK,
12+ **conn_params):
13+ if max_retries == requests.adapters.DEFAULT_RETRIES:
14+ self.max_retries = requests.adapters.Retry(0, read=False)
15+ else:
16+ self.max_retries = requests.adapters.Retry.from_int(max_retries)
17+ self.config = {}
18+ self.proxy_manager = {}
19+
20+ super(requests.adapters.HTTPAdapter, self).__init__()
21+
22+ self._pool_connections = pool_connections
23+ self._pool_maxsize = pool_maxsize
24+ self._pool_block = pool_block
25+ self._conn_params = conn_params
26+
27+ self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block, **conn_params)
28+
29+ def __setstate__(self, state):
30+ # Can't handle by adding 'proxy_manager' to self.__attrs__ because
31+ # because self.poolmanager uses a lambda function, which isn't pickleable.
32+ self.proxy_manager = {}
33+ self.config = {}
34+
35+ for attr, value in state.items():
36+ setattr(self, attr, value)
37+
38+ self.init_poolmanager(self._pool_connections, self._pool_maxsize,
39+ block=self._pool_block, **self._conn_params)
40+
41+
42+if settings.get('source_ips'):
43+ http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
44+ for source_ip in settings['source_ips'])
45+ https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
46+ for source_ip in settings['source_ips'])
47+else:
48+ http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
49+ https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
650
751
852class SessionSinglePool(requests.Session):
953
1054 def __init__(self):
11− global the_https_adapter, the_http_adapter
1255 super(SessionSinglePool, self).__init__()
1356
1457 # reuse the same adapters
1558 self.adapters.clear()
16− self.mount('https://', the_https_adapter)
17− self.mount('http://', the_http_adapter)
59+ self.mount('https://', next(https_adapters))
60+ self.mount('http://', next(http_adapters))
1861
1962 def close(self):
2063 """Call super, but clear adapters since there are managed globaly"""