aec6a26 · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #2319 from dalf/multiple-proxies

[enh] Add multiple outgoing proxies

commitaec6a2656c4af5aa307914b78cb4f46f4238109a authorAlexandre Flament <alex@al-f.net> dateMon 23 Nov 2020, 08:43 (6 years ago) parent2fc3b17 3786920

7 files changed+174−38

compare with master

unifiedsplit

docs/admin/settings.rst+35−17 searx/engines/__init__.py+10−8 searx/poolrequests.py+28−2 searx/search.py+1−1 searx/settings.yml+9−7 searx/webapp.py+2−3 tests/unit/test_poolrequests.py+89

docs/admin/settings.rst+35−17

@@ -36,18 +36,26 @@ Global Settings

3636 image_proxy : False # proxying image results through searx

3737 default_locale : "" # default interface locale

3838

39− # uncomment below section if you want to use a proxy

40−

41− #outgoing_proxies :

42− # http : http://127.0.0.1:8080

43− # https: http://127.0.0.1:8080

44−

45− # uncomment below section only if you have more than one network interface

46− # which can be the source of outgoing search requests

47−

48− #source_ips:

49− # - 1.1.1.1

50− # - 1.1.1.2

39+ outgoing: # communication with search engines

40+ request_timeout : 2.0 # default timeout in seconds, can be override by engine

41+ # max_request_timeout: 10.0 # the maximum timeout in seconds

42+ useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator

43+ pool_connections : 100 # Number of different hosts

44+ pool_maxsize : 10 # Number of simultaneous requests by host

45+

46+ #proxies:

47+ # http:

48+ # - http://proxy1:8080

49+ # - http://proxy2:8080

50+ # https:

51+ # - http://proxy1:8080

52+ # - http://proxy2:8080

53+ # - socks5://user:password@proxy3:1080

54+ # - socks5h://user:password@proxy4:1080

55+

56+ #source_ips:

57+ # - 1.1.1.1

58+ # - 1.1.1.2

5159

5260 locales:

5361 en : English

@@ -105,15 +113,16 @@ Global Settings

105113 code, like ``fr``, ``en``, ``de``.

106114

107115.. _requests proxies: http://requests.readthedocs.io/en/latest/user/advanced/#proxies

108−.. _PR SOCKS support: https://github.com/kennethreitz/requests/pull/478

116+.. _PySocks: https://pypi.org/project/PySocks/

109117

110−``outgoing_proxies`` :

111− Define a proxy you wish to use, see `requests proxies`_. SOCKS proxies are

112− not supported / see `PR SOCKS support`.

118+``proxies`` :

119+ Define one or more proxies you wish to use, see `requests proxies`_.

120+ If there are more than one proxy for one protocol (http, https),

121+ requests to the engines are distributed in a round-robin fashion.

113122

114123``source_ips`` :

115124 If you use multiple network interfaces, define from which IP the requests must

116− be made.

125+ be made. This parameter is ignored when ``proxies`` is set.

117126

118127``locales`` :

119128 Locales codes and their names. Available translations of searx interface.

@@ -139,6 +148,15 @@ Engine settings

139148 api_key : 'apikey'

140149 disabled : True

141150 language : en_US

151+ #proxies:

152+ # http:

153+ # - http://proxy1:8080

154+ # - http://proxy2:8080

155+ # https:

156+ # - http://proxy1:8080

157+ # - http://proxy2:8080

158+ # - socks5://user:password@proxy3:1080

159+ # - socks5h://user:password@proxy4:1080

142160

143161``name`` :

144162 Name that will be used across searx to define this engine. In settings, on

searx/engines/__init__.py+10−8

@@ -25,7 +25,7 @@ from operator import itemgetter

2525from searx import settings

2626from searx import logger

2727from searx.data import ENGINES_LANGUAGES

28−from searx.poolrequests import get

28+from searx.poolrequests import get, get_proxy_cycles

2929from searx.utils import load_module, match_language, get_engine_from_settings

3030

3131

@@ -79,16 +79,18 @@ def load_engine(engine_data):

7979 logger.exception('Cannot load engine "{}"'.format(engine_module))

8080 return None

8181

82− for param_name in engine_data:

82+ for param_name, param_value in engine_data.items():

8383 if param_name == 'engine':

84− continue

85− if param_name == 'categories':

86− if engine_data['categories'] == 'none':

84+ pass

85+ elif param_name == 'categories':

86+ if param_value == 'none':

8787 engine.categories = []

8888 else:

89− engine.categories = list(map(str.strip, engine_data['categories'].split(',')))

90− continue

91− setattr(engine, param_name, engine_data[param_name])

89+ engine.categories = list(map(str.strip, param_value.split(',')))

90+ elif param_name == 'proxies':

91+ engine.proxies = get_proxy_cycles(param_value)

92+ else:

93+ setattr(engine, param_name, param_value)

9294

9395 for arg_name, arg_value in engine_default_args.items():

9496 if not hasattr(engine, arg_name):

searx/poolrequests.py+28−2

@@ -111,6 +111,32 @@ def get_time_for_thread():

111111 return threadLocal.total_time

112112

113113

114+def get_proxy_cycles(proxy_settings):

115+ if not proxy_settings:

116+ return None

117+ # Backwards compatibility for single proxy in settings.yml

118+ for protocol, proxy in proxy_settings.items():

119+ if isinstance(proxy, str):

120+ proxy_settings[protocol] = [proxy]

121+

122+ for protocol in proxy_settings:

123+ proxy_settings[protocol] = cycle(proxy_settings[protocol])

124+ return proxy_settings

125+

126+

127+GLOBAL_PROXY_CYCLES = get_proxy_cycles(settings['outgoing'].get('proxies'))

128+

129+

130+def get_proxies(proxy_cycles):

131+ if proxy_cycles:

132+ return {protocol: next(proxy_cycle) for protocol, proxy_cycle in proxy_cycles.items()}

133+ return None

134+

135+

136+def get_global_proxies():

137+ return get_proxies(GLOBAL_PROXY_CYCLES)

138+

139+

114140def request(method, url, **kwargs):

115141 """same as requests/requests/api.py request(...)"""

116142 time_before_request = time()

@@ -119,8 +145,8 @@ def request(method, url, **kwargs):

119145 session = SessionSinglePool()

120146

121147 # proxies

122− if kwargs.get('proxies') is None:

123− kwargs['proxies'] = settings['outgoing'].get('proxies')

148+ if not kwargs.get('proxies'):

149+ kwargs['proxies'] = get_global_proxies()

124150

125151 # timeout

126152 if 'timeout' in kwargs:

searx/search.py+1−1

@@ -119,7 +119,7 @@ def send_http_request(engine, request_params):

119119

120120 # setting engine based proxies

121121 if hasattr(engine, 'proxies'):

122− request_args['proxies'] = engine.proxies

122+ request_args['proxies'] = requests_lib.get_proxies(engine.proxies)

123123

124124 # specific type of request (GET or POST)

125125 if request_params['method'] == 'GET':

searx/settings.yml+9−7

@@ -63,13 +63,15 @@ outgoing: # communication with search engines

6363 pool_connections : 100 # Number of different hosts

6464 pool_maxsize : 10 # Number of simultaneous requests by host

6565# uncomment below section if you want to use a proxy

66−# see http://docs.python-requests.org/en/latest/user/advanced/#proxies

67−# SOCKS proxies are also supported: see http://requests.readthedocs.io/en/master/user/advanced/#socks

68−# proxies :

69−# http : socks5h://127.0.0.1:9050

70−# https: socks5h://127.0.0.1:9050

71−# using_tor_proxy : True

72−# extra_proxy_timeout : 10.0 # Extra seconds to add in order to account for the time taken by the proxy

66+# see https://2.python-requests.org/en/latest/user/advanced/#proxies

67+# SOCKS proxies are also supported: see https://2.python-requests.org/en/latest/user/advanced/#socks

68+# proxies:

69+# http:

70+# - http://proxy1:8080

71+# - http://proxy2:8080

72+# https:

73+# - http://proxy1:8080

74+# - http://proxy2:8080

7375# uncomment below section only if you have more than one network interface

7476# which can be the source of outgoing search requests

7577# source_ips:

searx/webapp.py+2−3

@@ -78,6 +78,7 @@ from searx.plugins import plugins

7878from searx.plugins.oa_doi_rewrite import get_doi_resolver

7979from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

8080from searx.answerers import answerers

81+from searx.poolrequests import get_global_proxies

8182

8283

8384# serve pages with HTTP/1.1

@@ -149,8 +150,6 @@ _category_names = (gettext('files'),

149150 gettext('onions'),

150151 gettext('science'))

151152

152−outgoing_proxies = settings['outgoing'].get('proxies') or None

153−

154153_flask_babel_get_translations = flask_babel.get_translations

155154

156155

@@ -905,7 +904,7 @@ def image_proxy():

905904 stream=True,

906905 timeout=settings['outgoing']['request_timeout'],

907906 headers=headers,

908− proxies=outgoing_proxies)

907+ proxies=get_global_proxies())

909908

910909 if resp.status_code == 304:

911910 return '', resp.status_code

tests/unit/test_poolrequests.pynew file+89

@@ -0,0 +1,89 @@

1+from unittest.mock import patch

2+from requests.models import Response

4+from searx.testing import SearxTestCase

6+import searx.poolrequests

7+from searx.poolrequests import get_proxy_cycles, get_proxies

10+CONFIG = {'http': ['http://localhost:9090', 'http://localhost:9092'],

11+ 'https': ['http://localhost:9091', 'http://localhost:9093']}

12+

13+

14+class TestProxy(SearxTestCase):

15+

16+ def test_noconfig(self):

17+ cycles = get_proxy_cycles(None)

18+ self.assertIsNone(cycles)

19+

20+ cycles = get_proxy_cycles(False)

21+ self.assertIsNone(cycles)

22+

23+ def test_oldconfig(self):

24+ config = {

25+ 'http': 'http://localhost:9090',

26+ 'https': 'http://localhost:9091',

27+ }

28+ cycles = get_proxy_cycles(config)

29+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

30+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

31+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

32+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

33+

34+ def test_one_proxy(self):

35+ config = {

36+ 'http': ['http://localhost:9090'],

37+ 'https': ['http://localhost:9091'],

38+ }

39+ cycles = get_proxy_cycles(config)

40+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

41+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

42+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

43+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

44+

45+ def test_multiple_proxies(self):

46+ cycles = get_proxy_cycles(CONFIG)

47+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

48+ self.assertEqual(next(cycles['http']), 'http://localhost:9092')

49+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

50+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

51+ self.assertEqual(next(cycles['https']), 'http://localhost:9093')

52+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

53+

54+ def test_getproxies_none(self):

55+ self.assertIsNone(get_proxies(None))

56+

57+ def test_getproxies_config(self):

58+ cycles = get_proxy_cycles(CONFIG)

59+ self.assertEqual(get_proxies(cycles), {

60+ 'http': 'http://localhost:9090',

61+ 'https': 'http://localhost:9091'

62+ })

63+ self.assertEqual(get_proxies(cycles), {

64+ 'http': 'http://localhost:9092',

65+ 'https': 'http://localhost:9093'

66+ })

67+

68+ @patch('searx.poolrequests.get_global_proxies')

69+ def test_request(self, mock_get_global_proxies):

70+ method = 'GET'

71+ url = 'http://localhost'

72+ custom_proxies = {

73+ 'https': 'http://localhost:1080'

74+ }

75+ global_proxies = {

76+ 'http': 'http://localhost:9092',

77+ 'https': 'http://localhost:9093'

78+ }

79+ mock_get_global_proxies.return_value = global_proxies

80+

81+ # check the global proxies usage

82+ with patch.object(searx.poolrequests.SessionSinglePool, 'request', return_value=Response()) as mock_method:

83+ searx.poolrequests.request(method, url)

84+ mock_method.assert_called_once_with(method=method, url=url, proxies=global_proxies)

85+

86+ # check if the proxies parameter overrides the global proxies

87+ with patch.object(searx.poolrequests.SessionSinglePool, 'request', return_value=Response()) as mock_method:

88+ searx.poolrequests.request(method, url, proxies=custom_proxies)

89+ mock_method.assert_called_once_with(method=method, url=url, proxies=custom_proxies)