3786920 · searx · git.iko.soy

docs/admin/settings.rst+35−17

compare with master

unifiedsplit

@@ -36,18 +36,26 @@ Global Settings

36 image_proxy : False # proxying image results through searx

37 default_locale : "" # default interface locale

38

39− # uncomment below section if you want to use a proxy

39+ outgoing: # communication with search engines

40−

40+ request_timeout : 2.0 # default timeout in seconds, can be override by engine

41− #outgoing_proxies :

41+ # max_request_timeout: 10.0 # the maximum timeout in seconds

42− # http : http://127.0.0.1:8080

42+ useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator

43− # https: http://127.0.0.1:8080

43+ pool_connections : 100 # Number of different hosts

44−

44+ pool_maxsize : 10 # Number of simultaneous requests by host

45− # uncomment below section only if you have more than one network interface

45+

46− # which can be the source of outgoing search requests

46+ #proxies:

47−

47+ # http:

48+ # - http://proxy1:8080

49+ # - http://proxy2:8080

50+ # https:

51+ # - http://proxy1:8080

52+ # - http://proxy2:8080

53+ # - socks5://user:password@proxy3:1080

54+ # - socks5h://user:password@proxy4:1080

55+

48− #source_ips:

56+ #source_ips:

49− # - 1.1.1.1

57+ # - 1.1.1.1

50− # - 1.1.1.2

58+ # - 1.1.1.2

51

59

52 locales:

60 locales:

53 en : English

61 en : English

@@ -105,15 +113,16 @@ Global Settings

105 code, like ``fr``, ``en``, ``de``.

113 code, like ``fr``, ``en``, ``de``.

106

114

107.. _requests proxies: http://requests.readthedocs.io/en/latest/user/advanced/#proxies

115.. _requests proxies: http://requests.readthedocs.io/en/latest/user/advanced/#proxies

108−.. _PR SOCKS support: https://github.com/kennethreitz/requests/pull/478

116+.. _PySocks: https://pypi.org/project/PySocks/

109

117

110−``outgoing_proxies`` :

118+``proxies`` :

111− Define a proxy you wish to use, see `requests proxies`_. SOCKS proxies are

119+ Define one or more proxies you wish to use, see `requests proxies`_.

112− not supported / see `PR SOCKS support`.

120+ If there are more than one proxy for one protocol (http, https),

121+ requests to the engines are distributed in a round-robin fashion.

113

122

114``source_ips`` :

123``source_ips`` :

115 If you use multiple network interfaces, define from which IP the requests must

124 If you use multiple network interfaces, define from which IP the requests must

116− be made.

125+ be made. This parameter is ignored when ``proxies`` is set.

117

126

118``locales`` :

127``locales`` :

119 Locales codes and their names. Available translations of searx interface.

128 Locales codes and their names. Available translations of searx interface.

@@ -139,6 +148,15 @@ Engine settings

139 api_key : 'apikey'

148 api_key : 'apikey'

140 disabled : True

149 disabled : True

141 language : en_US

150 language : en_US

151+ #proxies:

152+ # http:

153+ # - http://proxy1:8080

154+ # - http://proxy2:8080

155+ # https:

156+ # - http://proxy1:8080

157+ # - http://proxy2:8080

158+ # - socks5://user:password@proxy3:1080

159+ # - socks5h://user:password@proxy4:1080

142

160

143``name`` :

161``name`` :

144 Name that will be used across searx to define this engine. In settings, on

162 Name that will be used across searx to define this engine. In settings, on

searx/engines/__init__.py+10−8

@@ -25,7 +25,7 @@ from operator import itemgetter

25from searx import settings

26from searx import logger

27from searx.data import ENGINES_LANGUAGES

28−from searx.poolrequests import get

28+from searx.poolrequests import get, get_proxy_cycles

29from searx.utils import load_module, match_language, get_engine_from_settings

30

31

@@ -79,16 +79,18 @@ def load_engine(engine_data):

79 logger.exception('Cannot load engine "{}"'.format(engine_module))

80 return None

81

82− for param_name in engine_data:

82+ for param_name, param_value in engine_data.items():

83 if param_name == 'engine':

84− continue

84+ pass

85− if param_name == 'categories':

85+ elif param_name == 'categories':

86− if engine_data['categories'] == 'none':

86+ if param_value == 'none':

87 engine.categories = []

88 else:

89− engine.categories = list(map(str.strip, engine_data['categories'].split(',')))

89+ engine.categories = list(map(str.strip, param_value.split(',')))

90− continue

90+ elif param_name == 'proxies':

91+ engine.proxies = get_proxy_cycles(param_value)

92+ else:

91− setattr(engine, param_name, engine_data[param_name])

93+ setattr(engine, param_name, param_value)

92

94

93 for arg_name, arg_value in engine_default_args.items():

95 for arg_name, arg_value in engine_default_args.items():

94 if not hasattr(engine, arg_name):

96 if not hasattr(engine, arg_name):

searx/poolrequests.py+28−2

@@ -111,6 +111,32 @@ def get_time_for_thread():

111 return threadLocal.total_time

112

113

114+def get_proxy_cycles(proxy_settings):

115+ if not proxy_settings:

116+ return None

117+ # Backwards compatibility for single proxy in settings.yml

118+ for protocol, proxy in proxy_settings.items():

119+ if isinstance(proxy, str):

120+ proxy_settings[protocol] = [proxy]

121+

122+ for protocol in proxy_settings:

123+ proxy_settings[protocol] = cycle(proxy_settings[protocol])

124+ return proxy_settings

125+

126+

127+GLOBAL_PROXY_CYCLES = get_proxy_cycles(settings['outgoing'].get('proxies'))

128+

129+

130+def get_proxies(proxy_cycles):

131+ if proxy_cycles:

132+ return {protocol: next(proxy_cycle) for protocol, proxy_cycle in proxy_cycles.items()}

133+ return None

134+

135+

136+def get_global_proxies():

137+ return get_proxies(GLOBAL_PROXY_CYCLES)

138+

139+

114def request(method, url, **kwargs):

140def request(method, url, **kwargs):

115 """same as requests/requests/api.py request(...)"""

141 """same as requests/requests/api.py request(...)"""

116 time_before_request = time()

142 time_before_request = time()

@@ -119,8 +145,8 @@ def request(method, url, **kwargs):

119 session = SessionSinglePool()

145 session = SessionSinglePool()

120

146

121 # proxies

147 # proxies

122− if kwargs.get('proxies') is None:

148+ if not kwargs.get('proxies'):

123− kwargs['proxies'] = settings['outgoing'].get('proxies')

149+ kwargs['proxies'] = get_global_proxies()

124

150

125 # timeout

151 # timeout

126 if 'timeout' in kwargs:

152 if 'timeout' in kwargs:

searx/search.py+1−1

@@ -119,7 +119,7 @@ def send_http_request(engine, request_params):

119

120 # setting engine based proxies

121 if hasattr(engine, 'proxies'):

122− request_args['proxies'] = engine.proxies

122+ request_args['proxies'] = requests_lib.get_proxies(engine.proxies)

123

124 # specific type of request (GET or POST)

125 if request_params['method'] == 'GET':

searx/settings.yml+9−7

@@ -63,13 +63,15 @@ outgoing: # communication with search engines

63 pool_connections : 100 # Number of different hosts

64 pool_maxsize : 10 # Number of simultaneous requests by host

65# uncomment below section if you want to use a proxy

66−# see http://docs.python-requests.org/en/latest/user/advanced/#proxies

66+# see https://2.python-requests.org/en/latest/user/advanced/#proxies

67−# SOCKS proxies are also supported: see http://requests.readthedocs.io/en/master/user/advanced/#socks

67+# SOCKS proxies are also supported: see https://2.python-requests.org/en/latest/user/advanced/#socks

68−# proxies :

68+# proxies:

69−# http : socks5h://127.0.0.1:9050

69+# http:

70−# https: socks5h://127.0.0.1:9050

70+# - http://proxy1:8080

71−# using_tor_proxy : True

71+# - http://proxy2:8080

72−# extra_proxy_timeout : 10.0 # Extra seconds to add in order to account for the time taken by the proxy

72+# https:

73+# - http://proxy1:8080

74+# - http://proxy2:8080

73# uncomment below section only if you have more than one network interface

75# uncomment below section only if you have more than one network interface

74# which can be the source of outgoing search requests

76# which can be the source of outgoing search requests

75# source_ips:

77# source_ips:

searx/webapp.py+2−3

@@ -78,6 +78,7 @@ from searx.plugins import plugins

78from searx.plugins.oa_doi_rewrite import get_doi_resolver

79from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

80from searx.answerers import answerers

81+from searx.poolrequests import get_global_proxies

81

82

83

83# serve pages with HTTP/1.1

84# serve pages with HTTP/1.1

@@ -149,8 +150,6 @@ _category_names = (gettext('files'),

149 gettext('onions'),

150 gettext('onions'),

150 gettext('science'))

151 gettext('science'))

151

152

152−outgoing_proxies = settings['outgoing'].get('proxies') or None

153−

154_flask_babel_get_translations = flask_babel.get_translations

153_flask_babel_get_translations = flask_babel.get_translations

155

154

156

155

@@ -905,7 +904,7 @@ def image_proxy():

905 stream=True,

904 stream=True,

906 timeout=settings['outgoing']['request_timeout'],

905 timeout=settings['outgoing']['request_timeout'],

907 headers=headers,

906 headers=headers,

908− proxies=outgoing_proxies)

907+ proxies=get_global_proxies())

909

908

910 if resp.status_code == 304:

909 if resp.status_code == 304:

911 return '', resp.status_code

910 return '', resp.status_code

tests/unit/test_poolrequests.pynew file+89

@@ -0,0 +1,89 @@

1+from unittest.mock import patch

2+from requests.models import Response

3+

4+from searx.testing import SearxTestCase

5+

6+import searx.poolrequests

7+from searx.poolrequests import get_proxy_cycles, get_proxies

8+

9+

10+CONFIG = {'http': ['http://localhost:9090', 'http://localhost:9092'],

11+ 'https': ['http://localhost:9091', 'http://localhost:9093']}

12+

13+

14+class TestProxy(SearxTestCase):

15+

16+ def test_noconfig(self):

17+ cycles = get_proxy_cycles(None)

18+ self.assertIsNone(cycles)

19+

20+ cycles = get_proxy_cycles(False)

21+ self.assertIsNone(cycles)

22+

23+ def test_oldconfig(self):

24+ config = {

25+ 'http': 'http://localhost:9090',

26+ 'https': 'http://localhost:9091',

27+ }

28+ cycles = get_proxy_cycles(config)

29+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

30+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

31+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

32+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

33+

34+ def test_one_proxy(self):

35+ config = {

36+ 'http': ['http://localhost:9090'],

37+ 'https': ['http://localhost:9091'],

38+ }

39+ cycles = get_proxy_cycles(config)

40+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

41+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

42+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

43+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

44+

45+ def test_multiple_proxies(self):

46+ cycles = get_proxy_cycles(CONFIG)

47+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

48+ self.assertEqual(next(cycles['http']), 'http://localhost:9092')

49+ self.assertEqual(next(cycles['http']), 'http://localhost:9090')

50+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

51+ self.assertEqual(next(cycles['https']), 'http://localhost:9093')

52+ self.assertEqual(next(cycles['https']), 'http://localhost:9091')

53+

54+ def test_getproxies_none(self):

55+ self.assertIsNone(get_proxies(None))

56+

57+ def test_getproxies_config(self):

58+ cycles = get_proxy_cycles(CONFIG)

59+ self.assertEqual(get_proxies(cycles), {

60+ 'http': 'http://localhost:9090',

61+ 'https': 'http://localhost:9091'

62+ })

63+ self.assertEqual(get_proxies(cycles), {

64+ 'http': 'http://localhost:9092',

65+ 'https': 'http://localhost:9093'

66+ })

67+

68+ @patch('searx.poolrequests.get_global_proxies')

69+ def test_request(self, mock_get_global_proxies):

70+ method = 'GET'

71+ url = 'http://localhost'

72+ custom_proxies = {

73+ 'https': 'http://localhost:1080'

74+ }

75+ global_proxies = {

76+ 'http': 'http://localhost:9092',

77+ 'https': 'http://localhost:9093'

78+ }

79+ mock_get_global_proxies.return_value = global_proxies

80+

81+ # check the global proxies usage

82+ with patch.object(searx.poolrequests.SessionSinglePool, 'request', return_value=Response()) as mock_method:

83+ searx.poolrequests.request(method, url)

84+ mock_method.assert_called_once_with(method=method, url=url, proxies=global_proxies)

85+

86+ # check if the proxies parameter overrides the global proxies

87+ with patch.object(searx.poolrequests.SessionSinglePool, 'request', return_value=Response()) as mock_method:

88+ searx.poolrequests.request(method, url, proxies=custom_proxies)

89+ mock_method.assert_called_once_with(method=method, url=url, proxies=custom_proxies)