git.iko.soy/searx
on master help sign in
[enh] support of multiple outgoing IPs
03579c2Adam Tauber
searx/poolrequests.py+48−5
compare with master
unifiedsplit
@@ -1,20 +1,63 @@
1import requests
1import requests
2+from itertools import cycle
3+from searx import settings
2
4
3
5
4−the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
6+class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):
5−the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
7+
8+ def __init__(self, pool_connections=requests.adapters.DEFAULT_POOLSIZE,
9+ pool_maxsize=requests.adapters.DEFAULT_POOLSIZE,
10+ max_retries=requests.adapters.DEFAULT_RETRIES,
11+ pool_block=requests.adapters.DEFAULT_POOLBLOCK,
12+ **conn_params):
13+ if max_retries == requests.adapters.DEFAULT_RETRIES:
14+ self.max_retries = requests.adapters.Retry(0, read=False)
15+ else:
16+ self.max_retries = requests.adapters.Retry.from_int(max_retries)
17+ self.config = {}
18+ self.proxy_manager = {}
19+
20+ super(requests.adapters.HTTPAdapter, self).__init__()
21+
22+ self._pool_connections = pool_connections
23+ self._pool_maxsize = pool_maxsize
24+ self._pool_block = pool_block
25+ self._conn_params = conn_params
26+
27+ self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block, **conn_params)
28+
29+ def __setstate__(self, state):
30+ # Can't handle by adding 'proxy_manager' to self.__attrs__ because
31+ # because self.poolmanager uses a lambda function, which isn't pickleable.
32+ self.proxy_manager = {}
33+ self.config = {}
34+
35+ for attr, value in state.items():
36+ setattr(self, attr, value)
37+
38+ self.init_poolmanager(self._pool_connections, self._pool_maxsize,
39+ block=self._pool_block, **self._conn_params)
40+
41+
42+if settings.get('source_ips'):
43+ http_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
44+ for source_ip in settings['source_ips'])
45+ https_adapters = cycle(HTTPAdapterWithConnParams(pool_connections=100, source_address=(source_ip, 0))
46+ for source_ip in settings['source_ips'])
47+else:
48+ http_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
49+ https_adapters = cycle((HTTPAdapterWithConnParams(pool_connections=100), ))
6
50
7
51
8class SessionSinglePool(requests.Session):
52class SessionSinglePool(requests.Session):
9
53
10 def __init__(self):
54 def __init__(self):
11− global the_https_adapter, the_http_adapter
12 super(SessionSinglePool, self).__init__()
55 super(SessionSinglePool, self).__init__()
13
56
14 # reuse the same adapters
57 # reuse the same adapters
15 self.adapters.clear()
58 self.adapters.clear()
16− self.mount('https://', the_https_adapter)
59+ self.mount('https://', next(https_adapters))
17− self.mount('http://', the_http_adapter)
60+ self.mount('http://', next(http_adapters))
18
61
19 def close(self):
62 def close(self):
20 """Call super, but clear adapters since there are managed globaly"""
63 """Call super, but clear adapters since there are managed globaly"""
searx/settings.yml+6
@@ -11,6 +11,12 @@ server:
11 image_proxy : False # Proxying image results through searx
11 image_proxy : False # Proxying image results through searx
12 default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
12 default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
13
13
14+# uncomment below section only if you have more than one network interface
15+# which can be the source of outgoing search requests
16+#source_ips:
17+# - 1.1.1.1
18+# - 1.1.1.2
19+
14engines:
20engines:
15 - name : wikipedia
21 - name : wikipedia
16 engine : mediawiki
22 engine : mediawiki