3062e2a · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

Merge pull request #1823 from kvch/make-engines-private

3062e2aMarkus Heiser7 years ago

searx/__init__.py+1

compare with master

unifiedsplit

@@ -38,6 +38,7 @@ def check_settings_yml(file_name):

38 else:

39 return None

41+

41# find location of settings.yml

42# find location of settings.yml

42if 'SEARX_SETTINGS_PATH' in environ:

43if 'SEARX_SETTINGS_PATH' in environ:

43 # if possible set path to settings using the

44 # if possible set path to settings using the

searx/engines/__init__.py+7−2

@@ -54,7 +54,8 @@ engine_default_args = {'paging': False,

54 'suspend_end_time': 0,

55 'continuous_errors': 0,

56 'time_range_support': False,

57− 'offline': False}

57+ 'offline': False,

58+ 'tokens': []}

60def load_engine(engine_data):

61def load_engine(engine_data):

@@ -160,7 +161,7 @@ def to_percentage(stats, maxvalue):

160 return stats

161 return stats

161

162

163

163−def get_engines_stats():

164+def get_engines_stats(preferences):

164 # TODO refactor

165 # TODO refactor

165 pageloads = []

166 pageloads = []

166 engine_times = []

167 engine_times = []

@@ -171,8 +172,12 @@ def get_engines_stats():

171

172

172 max_pageload = max_engine_times = max_results = max_score = max_errors = max_score_per_result = 0 # noqa

173 max_pageload = max_engine_times = max_results = max_score = max_errors = max_score_per_result = 0 # noqa

173 for engine in engines.values():

174 for engine in engines.values():

175+ if not preferences.validate_token(engine):

176+ continue

177+

174 if engine.stats['search_count'] == 0:

178 if engine.stats['search_count'] == 0:

175 continue

179 continue

180+

176 results_num = \

181 results_num = \

177 engine.stats['result_count'] / float(engine.stats['search_count'])

182 engine.stats['result_count'] / float(engine.stats['search_count'])

178

183

searx/engines/dummy-offline.pynew file+12

@@ -0,0 +1,12 @@

1+"""

2+ Dummy Offline

4+ @results one result

5+ @stable yes

6+"""

9+def search(query, request_params):

10+ return [{

11+ 'result': 'this is what you get',

12+ }]

searx/engines/genius.py+1

@@ -72,6 +72,7 @@ def parse_album(hit):

72 result.update({'content': 'Released: {}'.format(year)})

73 return result

75+

75parse = {'lyric': parse_lyric, 'song': parse_lyric, 'artist': parse_artist, 'album': parse_album}

76parse = {'lyric': parse_lyric, 'song': parse_lyric, 'artist': parse_artist, 'album': parse_album}

searx/preferences.py+44

@@ -104,6 +104,31 @@ class MultipleChoiceSetting(EnumStringSetting):

104 resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE)

105

106

107+class SetSetting(Setting):

108+ def _post_init(self):

109+ if not hasattr(self, 'values'):

110+ self.values = set()

111+

112+ def get_value(self):

113+ return ','.join(self.values)

114+

115+ def parse(self, data):

116+ if data == '':

117+ self.values = set()

118+ return

119+

120+ elements = data.split(',')

121+ for element in elements:

122+ self.values.add(element)

123+

124+ def parse_form(self, data):

125+ elements = data.split(',')

126+ self.values = set(elements)

127+

128+ def save(self, name, resp):

129+ resp.set_cookie(name, ','.join(self.values), max_age=COOKIE_MAX_AGE)

130+

131+

107class SearchLanguageSetting(EnumStringSetting):

132class SearchLanguageSetting(EnumStringSetting):

108 """Available choices may change, so user's value may not be in choices anymore"""

133 """Available choices may change, so user's value may not be in choices anymore"""

109

134

@@ -272,6 +297,7 @@ class Preferences(object):

272

297

273 self.engines = EnginesSetting('engines', choices=engines)

298 self.engines = EnginesSetting('engines', choices=engines)

274 self.plugins = PluginsSetting('plugins', choices=plugins)

299 self.plugins = PluginsSetting('plugins', choices=plugins)

300+ self.tokens = SetSetting('tokens')

275 self.unknown_params = {}

301 self.unknown_params = {}

276

302

277 def get_as_url_params(self):

303 def get_as_url_params(self):

@@ -288,6 +314,8 @@ class Preferences(object):

288 settings_kv['disabled_plugins'] = ','.join(self.plugins.disabled)

314 settings_kv['disabled_plugins'] = ','.join(self.plugins.disabled)

289 settings_kv['enabled_plugins'] = ','.join(self.plugins.enabled)

315 settings_kv['enabled_plugins'] = ','.join(self.plugins.enabled)

290

316

317+ settings_kv['tokens'] = ','.join(self.tokens.values)

318+

291 return urlsafe_b64encode(compress(urlencode(settings_kv).encode('utf-8'))).decode('utf-8')

319 return urlsafe_b64encode(compress(urlencode(settings_kv).encode('utf-8'))).decode('utf-8')

292

320

293 def parse_encoded_data(self, input_data):

321 def parse_encoded_data(self, input_data):

@@ -307,6 +335,8 @@ class Preferences(object):

307 elif user_setting_name == 'disabled_plugins':

335 elif user_setting_name == 'disabled_plugins':

308 self.plugins.parse_cookie((input_data.get('disabled_plugins', ''),

336 self.plugins.parse_cookie((input_data.get('disabled_plugins', ''),

309 input_data.get('enabled_plugins', '')))

337 input_data.get('enabled_plugins', '')))

338+ elif user_setting_name == 'tokens':

339+ self.tokens.parse(user_setting)

310 elif not any(user_setting_name.startswith(x) for x in [

340 elif not any(user_setting_name.startswith(x) for x in [

311 'enabled_',

341 'enabled_',

312 'disabled_',

342 'disabled_',

@@ -328,6 +358,8 @@ class Preferences(object):

328 enabled_categories.append(user_setting_name[len('category_'):])

358 enabled_categories.append(user_setting_name[len('category_'):])

329 elif user_setting_name.startswith('plugin_'):

359 elif user_setting_name.startswith('plugin_'):

330 disabled_plugins.append(user_setting_name)

360 disabled_plugins.append(user_setting_name)

361+ elif user_setting_name == 'tokens':

362+ self.tokens.parse_form(user_setting)

331 else:

363 else:

332 self.unknown_params[user_setting_name] = user_setting

364 self.unknown_params[user_setting_name] = user_setting

333 self.key_value_settings['categories'].parse_form(enabled_categories)

365 self.key_value_settings['categories'].parse_form(enabled_categories)

@@ -346,6 +378,18 @@ class Preferences(object):

346 user_setting.save(user_setting_name, resp)

378 user_setting.save(user_setting_name, resp)

347 self.engines.save(resp)

379 self.engines.save(resp)

348 self.plugins.save(resp)

380 self.plugins.save(resp)

381+ self.tokens.save('tokens', resp)

349 for k, v in self.unknown_params.items():

382 for k, v in self.unknown_params.items():

350 resp.set_cookie(k, v, max_age=COOKIE_MAX_AGE)

383 resp.set_cookie(k, v, max_age=COOKIE_MAX_AGE)

351 return resp

384 return resp

385+

386+ def validate_token(self, engine):

387+ valid = True

388+ if hasattr(engine, 'tokens') and engine.tokens:

389+ valid = False

390+ for token in self.tokens.values:

391+ if token in engine.tokens:

392+ valid = True

393+ break

394+

395+ return valid

searx/query.py+3−1

@@ -177,7 +177,8 @@ class RawTextQuery(object):

177class SearchQuery(object):

178 """container for all the search parameters (query, language, etc...)"""

179

180− def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range, timeout_limit=None):

180+ def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range,

181+ timeout_limit=None, preferences=None):

181 self.query = query.encode('utf-8')

182 self.query = query.encode('utf-8')

182 self.engines = engines

183 self.engines = engines

183 self.categories = categories

184 self.categories = categories

@@ -186,6 +187,7 @@ class SearchQuery(object):

186 self.pageno = pageno

187 self.pageno = pageno

187 self.time_range = None if time_range in ('', 'None', None) else time_range

188 self.time_range = None if time_range in ('', 'None', None) else time_range

188 self.timeout_limit = timeout_limit

189 self.timeout_limit = timeout_limit

190+ self.preferences = preferences

189

191

190 def __str__(self):

192 def __str__(self):

191 return str(self.query) + ";" + str(self.engines)

193 return str(self.query) + ";" + str(self.engines)

searx/search.py+4−1

@@ -407,7 +407,7 @@ def get_search_query_from_webapp(preferences, form):

407

408 return (SearchQuery(query, query_engines, query_categories,

409 query_lang, query_safesearch, query_pageno,

410− query_time_range, query_timeout),

410+ query_time_range, query_timeout, preferences),

411 raw_text_query)

412

413

@@ -459,6 +459,9 @@ class Search(object):

459

460 engine = engines[selected_engine['name']]

461

462+ if not search_query.preferences.validate_token(engine):

463+ continue

464+

462 # skip suspended engines

465 # skip suspended engines

463 if engine.suspend_end_time >= time():

466 if engine.suspend_end_time >= time():

464 logger.debug('Engine currently suspended: %s', selected_engine['name'])

467 logger.debug('Engine currently suspended: %s', selected_engine['name'])

searx/templates/oscar/preferences.html+6

@@ -131,6 +131,12 @@

131 {% endfor %}

132 </select>

133 {{ preferences_item_footer(info, label, rtl) }}

134+

135+ {% set label = _('Engine tokens') %}

136+ {% set info = _('Access tokens for private engines') %}

137+ {{ preferences_item_header(info, label, rtl) }}

138+ <input class="form-control" id='tokens' name='tokens' value='{{ preferences.tokens.get_value() }}'/>

139+ {{ preferences_item_footer(info, label, rtl) }}

134 </div>

140 </div>

135 </fieldset>

141 </fieldset>

136 </div>

142 </div>

searx/webapp.py+12−5

@@ -731,8 +731,13 @@ def preferences():

731 # stats for preferences page

732 stats = {}

733

734+ engines_by_category = {}

734 for c in categories:

735 for c in categories:

736+ engines_by_category[c] = []

735 for e in categories[c]:

737 for e in categories[c]:

738+ if not request.preferences.validate_token(e):

739+ continue

740+

736 stats[e.name] = {'time': None,

741 stats[e.name] = {'time': None,

737 'warn_timeout': False,

742 'warn_timeout': False,

738 'warn_time': False}

743 'warn_time': False}

@@ -740,9 +745,11 @@ def preferences():

740 stats[e.name]['warn_timeout'] = True

745 stats[e.name]['warn_timeout'] = True

741 stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)

746 stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)

742

747

748+ engines_by_category[c].append(e)

749+

743 # get first element [0], the engine time,

750 # get first element [0], the engine time,

744 # and then the second element [1] : the time (the first one is the label)

751 # and then the second element [1] : the time (the first one is the label)

745− for engine_stat in get_engines_stats()[0][1]:

752+ for engine_stat in get_engines_stats(request.preferences)[0][1]:

746 stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)

753 stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)

747 if engine_stat.get('avg') > settings['outgoing']['request_timeout']:

754 if engine_stat.get('avg') > settings['outgoing']['request_timeout']:

748 stats[engine_stat.get('name')]['warn_time'] = True

755 stats[engine_stat.get('name')]['warn_time'] = True

@@ -752,7 +759,7 @@ def preferences():

752 locales=settings['locales'],

759 locales=settings['locales'],

753 current_locale=get_locale(),

760 current_locale=get_locale(),

754 image_proxy=image_proxy,

761 image_proxy=image_proxy,

755− engines_by_category=categories,

762+ engines_by_category=engines_by_category,

756 stats=stats,

763 stats=stats,

757 answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],

764 answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],

758 disabled_engines=disabled_engines,

765 disabled_engines=disabled_engines,

@@ -828,7 +835,7 @@ def image_proxy():

828@app.route('/stats', methods=['GET'])

835@app.route('/stats', methods=['GET'])

829def stats():

836def stats():

830 """Render engine statistics page."""

837 """Render engine statistics page."""

831− stats = get_engines_stats()

838+ stats = get_engines_stats(request.preferences)

832 return render(

839 return render(

833 'stats.html',

840 'stats.html',

834 stats=stats,

841 stats=stats,

@@ -891,7 +898,7 @@ def clear_cookies():

891@app.route('/config')

898@app.route('/config')

892def config():

899def config():

893 return jsonify({'categories': list(categories.keys()),

900 return jsonify({'categories': list(categories.keys()),

894− 'engines': [{'name': engine_name,

901+ 'engines': [{'name': name,

895 'categories': engine.categories,

902 'categories': engine.categories,

896 'shortcut': engine.shortcut,

903 'shortcut': engine.shortcut,

897 'enabled': not engine.disabled,

904 'enabled': not engine.disabled,

@@ -904,7 +911,7 @@ def config():

904 'safesearch': engine.safesearch,

911 'safesearch': engine.safesearch,

905 'time_range_support': engine.time_range_support,

912 'time_range_support': engine.time_range_support,

906 'timeout': engine.timeout}

913 'timeout': engine.timeout}

907− for engine_name, engine in engines.items()],

914+ for name, engine in engines.items() if request.preferences.validate_token(engine)],

908 'plugins': [{'name': plugin.name,

915 'plugins': [{'name': plugin.name,

909 'enabled': plugin.default_on}

916 'enabled': plugin.default_on}

910 for plugin in plugins],

917 for plugin in plugins],

tests/unit/test_search.py+71−19

@@ -1,60 +1,112 @@

1# -*- coding: utf-8 -*-

3from searx.testing import SearxTestCase

4+from searx.preferences import Preferences

5+from searx.engines import engines

5−import searx.preferences

6import searx.search

7import searx.search

7−import searx.engines

10+SAFESEARCH = 0

11+PAGENO = 1

12+PUBLIC_ENGINE_NAME = 'general dummy'

13+PRIVATE_ENGINE_NAME = 'general private offline'

14+TEST_ENGINES = [

15+ {

16+ 'name': PUBLIC_ENGINE_NAME,

17+ 'engine': 'dummy',

18+ 'categories': 'general',

19+ 'shortcut': 'gd',

20+ 'timeout': 3.0,

21+ 'tokens': [],

22+ },

23+ {

24+ 'name': PRIVATE_ENGINE_NAME,

25+ 'engine': 'dummy-offline',

26+ 'categories': 'general',

27+ 'shortcut': 'do',

28+ 'timeout': 3.0,

29+ 'offline': True,

30+ 'tokens': ['my-token'],

31+ },

32+]

10class SearchTestCase(SearxTestCase):

35class SearchTestCase(SearxTestCase):

12 @classmethod

37 @classmethod

13 def setUpClass(cls):

38 def setUpClass(cls):

14− searx.engines.initialize_engines([{

39+ searx.engines.initialize_engines(TEST_ENGINES)

15− 'name': 'general dummy',

16− 'engine': 'dummy',

17− 'categories': 'general',

18− 'shortcut': 'gd',

19− 'timeout': 3.0

20− }])

22 def test_timeout_simple(self):

41 def test_timeout_simple(self):

23 searx.search.max_request_timeout = None

42 searx.search.max_request_timeout = None

24− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

43+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

25− ['general'], 'en-US', 0, 1, None, None)

44+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

45+ preferences=Preferences(['oscar'], ['general'], engines, []))

26 search = searx.search.Search(search_query)

46 search = searx.search.Search(search_query)

27 search.search()

47 search.search()

28 self.assertEquals(search.actual_timeout, 3.0)

48 self.assertEquals(search.actual_timeout, 3.0)

30 def test_timeout_query_above_default_nomax(self):

50 def test_timeout_query_above_default_nomax(self):

31 searx.search.max_request_timeout = None

51 searx.search.max_request_timeout = None

32− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

52+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

33− ['general'], 'en-US', 0, 1, None, 5.0)

53+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

54+ preferences=Preferences(['oscar'], ['general'], engines, []))

34 search = searx.search.Search(search_query)

55 search = searx.search.Search(search_query)

35 search.search()

56 search.search()

36 self.assertEquals(search.actual_timeout, 3.0)

57 self.assertEquals(search.actual_timeout, 3.0)

38 def test_timeout_query_below_default_nomax(self):

59 def test_timeout_query_below_default_nomax(self):

39 searx.search.max_request_timeout = None

60 searx.search.max_request_timeout = None

40− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

61+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

41− ['general'], 'en-US', 0, 1, None, 1.0)

62+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0,

63+ preferences=Preferences(['oscar'], ['general'], engines, []))

42 search = searx.search.Search(search_query)

64 search = searx.search.Search(search_query)

43 search.search()

65 search.search()

44 self.assertEquals(search.actual_timeout, 1.0)

66 self.assertEquals(search.actual_timeout, 1.0)

46 def test_timeout_query_below_max(self):

68 def test_timeout_query_below_max(self):

47 searx.search.max_request_timeout = 10.0

69 searx.search.max_request_timeout = 10.0

48− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

70+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

49− ['general'], 'en-US', 0, 1, None, 5.0)

71+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

72+ preferences=Preferences(['oscar'], ['general'], engines, []))

50 search = searx.search.Search(search_query)

73 search = searx.search.Search(search_query)

51 search.search()

74 search.search()

52 self.assertEquals(search.actual_timeout, 5.0)

75 self.assertEquals(search.actual_timeout, 5.0)

54 def test_timeout_query_above_max(self):

77 def test_timeout_query_above_max(self):

55 searx.search.max_request_timeout = 10.0

78 searx.search.max_request_timeout = 10.0

56− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

79+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

57− ['general'], 'en-US', 0, 1, None, 15.0)

80+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0,

81+ preferences=Preferences(['oscar'], ['general'], engines, []))

58 search = searx.search.Search(search_query)

82 search = searx.search.Search(search_query)

59 search.search()

83 search.search()

60 self.assertEquals(search.actual_timeout, 10.0)

84 self.assertEquals(search.actual_timeout, 10.0)

85+

86+ def test_query_private_engine_without_token(self):

87+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

88+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

89+ preferences=Preferences(['oscar'], ['general'], engines, []))

90+ search = searx.search.Search(search_query)

91+ results = search.search()

92+ self.assertEquals(results.results_length(), 0)

93+

94+ def test_query_private_engine_with_incorrect_token(self):

95+ preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

96+ preferences_with_tokens.parse_dict({'tokens': 'bad-token'})

97+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

98+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

99+ preferences=preferences_with_tokens)

100+ search = searx.search.Search(search_query)

101+ results = search.search()

102+ self.assertEquals(results.results_length(), 0)

103+

104+ def test_query_private_engine_with_correct_token(self):

105+ preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

106+ preferences_with_tokens.parse_dict({'tokens': 'my-token'})

107+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

108+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

109+ preferences=preferences_with_tokens)

110+ search = searx.search.Search(search_query)

111+ results = search.search()

112+ self.assertEquals(results.results_length(), 1)