bc60d83 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[enh] verify that Tor proxy works every time searx starts

bc60d83Alexandre Flament5 years ago

searx/network/__init__.py+2−2

compare with master

unifiedsplit

@@ -8,7 +8,7 @@ from time import time

8import httpx

9import h2.exceptions

11−from .network import get_network, initialize

11+from .network import get_network, initialize, check_network_configuration

12from .client import get_loop

13from .raise_for_httperror import raise_for_httperror

@@ -154,7 +154,7 @@ def delete(url, **kwargs):

154

155async def stream_chunk_to_queue(network, q, method, url, **kwargs):

156 try:

157− async with network.stream(method, url, **kwargs) as response:

157+ async with await network.stream(method, url, **kwargs) as response:

158 q.put(response)

159 async for chunk in response.aiter_bytes(65536):

160 if len(chunk) > 0:

searx/network/network.py+63−16

@@ -7,7 +7,7 @@ from itertools import cycle

8import httpx

10−from .client import new_client, get_loop

10+from .client import new_client, get_loop, AsyncHTTPTransportNoHttp

13DEFAULT_NAME = '__DEFAULT__'

@@ -36,9 +36,11 @@ class Network:

37 __slots__ = ('enable_http', 'verify', 'enable_http2',

38 'max_connections', 'max_keepalive_connections', 'keepalive_expiry',

39− 'local_addresses', 'proxies', 'max_redirects', 'retries', 'retry_on_http_error',

39+ 'local_addresses', 'proxies', 'using_tor_proxy', 'max_redirects', 'retries', 'retry_on_http_error',

40 '_local_addresses_cycle', '_proxies_cycle', '_clients')

42+ _TOR_CHECK_RESULT = {}

43+

42 def __init__(self,

44 def __init__(self,

43 enable_http=True,

45 enable_http=True,

44 verify=True,

46 verify=True,

@@ -47,6 +49,7 @@ class Network:

47 max_keepalive_connections=None,

49 max_keepalive_connections=None,

48 keepalive_expiry=None,

50 keepalive_expiry=None,

49 proxies=None,

51 proxies=None,

52+ using_tor_proxy=False,

50 local_addresses=None,

53 local_addresses=None,

51 retries=0,

54 retries=0,

52 retry_on_http_error=None,

55 retry_on_http_error=None,

@@ -58,6 +61,7 @@ class Network:

58 self.max_keepalive_connections = max_keepalive_connections

61 self.max_keepalive_connections = max_keepalive_connections

59 self.keepalive_expiry = keepalive_expiry

62 self.keepalive_expiry = keepalive_expiry

60 self.proxies = proxies

63 self.proxies = proxies

64+ self.using_tor_proxy = using_tor_proxy

61 self.local_addresses = local_addresses

65 self.local_addresses = local_addresses

62 self.retries = retries

66 self.retries = retries

63 self.retry_on_http_error = retry_on_http_error

67 self.retry_on_http_error = retry_on_http_error

@@ -121,23 +125,49 @@ class Network:

121 while True:

125 while True:

122 yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())

126 yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())

123

127

128+ @staticmethod

129+ async def check_tor_proxy(client: httpx.AsyncClient, proxies) -> bool:

130+ if proxies in Network._TOR_CHECK_RESULT:

131+ return Network._TOR_CHECK_RESULT[proxies]

132+

133+ result = True

134+ # ignore client._transport because it is not used with all://

135+ for transport in client._mounts.values(): # pylint: disable=protected-access

136+ if isinstance(transport, AsyncHTTPTransportNoHttp):

137+ continue

138+ if not getattr(transport, '_rdns', False):

139+ result = False

140+ break

141+ else:

142+ response = await client.get('https://check.torproject.org/api/ip')

143+ if not response.json()['IsTor']:

144+ result = False

145+ Network._TOR_CHECK_RESULT[proxies] = result

146+ return result

147+

124− def get_client(self, verify=None, max_redirects=None):

148+ async def get_client(self, verify=None, max_redirects=None):

125 verify = self.verify if verify is None else verify

149 verify = self.verify if verify is None else verify

126 max_redirects = self.max_redirects if max_redirects is None else max_redirects

150 max_redirects = self.max_redirects if max_redirects is None else max_redirects

127 local_address = next(self._local_addresses_cycle)

151 local_address = next(self._local_addresses_cycle)

128 proxies = next(self._proxies_cycle) # is a tuple so it can be part of the key

152 proxies = next(self._proxies_cycle) # is a tuple so it can be part of the key

129 key = (verify, max_redirects, local_address, proxies)

153 key = (verify, max_redirects, local_address, proxies)

130 if key not in self._clients or self._clients[key].is_closed:

154 if key not in self._clients or self._clients[key].is_closed:

155+ client = new_client(

131− self._clients[key] = new_client(self.enable_http,

156+ self.enable_http,

132− verify,

157+ verify,

133− self.enable_http2,

158+ self.enable_http2,

134− self.max_connections,

159+ self.max_connections,

135− self.max_keepalive_connections,

160+ self.max_keepalive_connections,

136− self.keepalive_expiry,

161+ self.keepalive_expiry,

137− dict(proxies),

162+ dict(proxies),

138− local_address,

163+ local_address,

139− 0,

164+ 0,

140− max_redirects)

165+ max_redirects

166+ )

167+ if self.using_tor_proxy and not await self.check_tor_proxy(client, proxies):

168+ await client.aclose()

169+ raise httpx.ProxyError('Network configuration problem: not using Tor')

170+ self._clients[key] = client

141 return self._clients[key]

171 return self._clients[key]

142

172

143 async def aclose(self):

173 async def aclose(self):

@@ -168,7 +198,7 @@ class Network:

168 retries = self.retries

198 retries = self.retries

169 while retries >= 0: # pragma: no cover

199 while retries >= 0: # pragma: no cover

170 kwargs_clients = Network.get_kwargs_clients(kwargs)

200 kwargs_clients = Network.get_kwargs_clients(kwargs)

171− client = self.get_client(**kwargs_clients)

201+ client = await self.get_client(**kwargs_clients)

172 try:

202 try:

173 response = await client.request(method, url, **kwargs)

203 response = await client.request(method, url, **kwargs)

174 if self.is_valid_respones(response) or retries <= 0:

204 if self.is_valid_respones(response) or retries <= 0:

@@ -178,11 +208,11 @@ class Network:

178 raise e

208 raise e

179 retries -= 1

209 retries -= 1

180

210

181− def stream(self, method, url, **kwargs):

211+ async def stream(self, method, url, **kwargs):

182 retries = self.retries

212 retries = self.retries

183 while retries >= 0: # pragma: no cover

213 while retries >= 0: # pragma: no cover

184 kwargs_clients = Network.get_kwargs_clients(kwargs)

214 kwargs_clients = Network.get_kwargs_clients(kwargs)

185− client = self.get_client(**kwargs_clients)

215+ client = await self.get_client(**kwargs_clients)

186 try:

216 try:

187 response = client.stream(method, url, **kwargs)

217 response = client.stream(method, url, **kwargs)

188 if self.is_valid_respones(response) or retries <= 0:

218 if self.is_valid_respones(response) or retries <= 0:

@@ -202,6 +232,22 @@ def get_network(name=None):

202 return NETWORKS[name or DEFAULT_NAME]

232 return NETWORKS[name or DEFAULT_NAME]

203

233

204

234

235+def check_network_configuration():

236+ async def check():

237+ exception_count = 0

238+ for network in NETWORKS.values():

239+ if network.using_tor_proxy:

240+ try:

241+ await network.get_client()

242+ except Exception: # pylint: disable=broad-except

243+ exception_count += 1

244+ return exception_count

245+ future = asyncio.run_coroutine_threadsafe(check(), get_loop())

246+ exception_count = future.result()

247+ if exception_count > 0:

248+ raise RuntimeError("Invalid network configuration")

249+

250+

205def initialize(settings_engines=None, settings_outgoing=None):

251def initialize(settings_engines=None, settings_outgoing=None):

206 from searx.engines import engines

252 from searx.engines import engines

207 from searx import settings

253 from searx import settings

@@ -225,6 +271,7 @@ def initialize(settings_engines=None, settings_outgoing=None):

225 'keepalive_expiry': settings_outgoing.get('keepalive_expiry', 5.0),

271 'keepalive_expiry': settings_outgoing.get('keepalive_expiry', 5.0),

226 'local_addresses': settings_outgoing.get('source_ips'),

272 'local_addresses': settings_outgoing.get('source_ips'),

227 'proxies': settings_outgoing.get('proxies'),

273 'proxies': settings_outgoing.get('proxies'),

274+ 'using_tor_proxy': settings_outgoing.get('using_tor_proxy'),

228 # default maximum redirect

275 # default maximum redirect

229 # from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55

276 # from https://github.com/psf/requests/blob/8c211a96cdbe9fe320d63d9e1ae15c5c07e179f8/requests/models.py#L55

230 'max_redirects': settings_outgoing.get('max_redirects', 30),

277 'max_redirects': settings_outgoing.get('max_redirects', 30),

searx/search/__init__.py+4−1

@@ -30,6 +30,7 @@ from searx import logger

30from searx.plugins import plugins

31from searx.search.models import EngineRef, SearchQuery

32from searx.search.processors import processors, initialize as initialize_processors

33+from searx.network import check_network_configuration

33from searx.search.checker import initialize as initialize_checker

34from searx.search.checker import initialize as initialize_checker

@@ -47,9 +48,11 @@ else:

47 sys.exit(1)

48 sys.exit(1)

50−def initialize(settings_engines=None, enable_checker=False):

51+def initialize(settings_engines=None, enable_checker=False, check_network=False):

51 settings_engines = settings_engines or settings['engines']

52 settings_engines = settings_engines or settings['engines']

52 initialize_processors(settings_engines)

53 initialize_processors(settings_engines)

54+ if check_network:

55+ check_network_configuration()

53 if enable_checker:

56 if enable_checker:

54 initialize_checker()

57 initialize_checker()

searx/webapp.py+1−1

@@ -153,7 +153,7 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai

153# initialize the engines except on the first run of the werkzeug server.

154if not werkzeug_reloader\

155 or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):

156− search_initialize(enable_checker=True)

156+ search_initialize(enable_checker=True, check_network=True)

157

158babel = Babel(app)

159

tests/unit/network/test_network.py+10−10

@@ -90,12 +90,12 @@ class TestNetwork(SearxTestCase):

91 async def test_get_client(self):

92 network = Network(verify=True)

93− client1 = network.get_client()

93+ client1 = await network.get_client()

94− client2 = network.get_client(verify=True)

94+ client2 = await network.get_client(verify=True)

95− client3 = network.get_client(max_redirects=10)

95+ client3 = await network.get_client(max_redirects=10)

96− client4 = network.get_client(verify=True)

96+ client4 = await network.get_client(verify=True)

97− client5 = network.get_client(verify=False)

97+ client5 = await network.get_client(verify=False)

98− client6 = network.get_client(max_redirects=10)

98+ client6 = await network.get_client(max_redirects=10)

100 self.assertEqual(client1, client2)

101 self.assertEqual(client1, client4)

@@ -107,7 +107,7 @@ class TestNetwork(SearxTestCase):

107

108 async def test_aclose(self):

109 network = Network(verify=True)

110− network.get_client()

110+ await network.get_client()

111 await network.aclose()

112

113 async def test_request(self):

@@ -211,7 +211,7 @@ class TestNetworkStreamRetries(SearxTestCase):

211 async def test_retries_ok(self):

212 with patch.object(httpx.AsyncClient, 'stream', new=TestNetworkStreamRetries.get_response_exception_then_200()):

213 network = Network(enable_http=True, retries=1, retry_on_http_error=403)

214− response = network.stream('GET', 'https://example.com/')

214+ response = await network.stream('GET', 'https://example.com/')

215 self.assertEqual(response.text, TestNetworkStreamRetries.TEXT)

216 await network.aclose()

217

@@ -219,7 +219,7 @@ class TestNetworkStreamRetries(SearxTestCase):

219 with patch.object(httpx.AsyncClient, 'stream', new=TestNetworkStreamRetries.get_response_exception_then_200()):

220 network = Network(enable_http=True, retries=0, retry_on_http_error=403)

221 with self.assertRaises(httpx.RequestError):

222− network.stream('GET', 'https://example.com/')

222+ await network.stream('GET', 'https://example.com/')

223 await network.aclose()

224

225 async def test_retries_exception(self):

@@ -234,6 +234,6 @@ class TestNetworkStreamRetries(SearxTestCase):

234

235 with patch.object(httpx.AsyncClient, 'stream', new=stream):

236 network = Network(enable_http=True, retries=0, retry_on_http_error=403)

237− response = network.stream('GET', 'https://example.com/')

237+ response = await network.stream('GET', 'https://example.com/')

238 self.assertEqual(response.status_code, 403)

239 await network.aclose()