9943538 · searx · git.iko.soy

git.iko.soy/searx

[enh] introduce private engines

This PR adds a new setting to engines named `tokens`. It expects a list of tokens which lets searx validate if the request should be accepted or not.

commit99435381a84072b110c32004b2fb778af9b96f77 authorNoémi Ványi <sitbackandwait@gmail.com> dateSat 1 Feb 2020, 11:01 (7 years ago) parentf9c7a67

10 files changed+161−28

compare with master

unifiedsplit

searx/__init__.py+1 searx/engines/__init__.py+7−2 searx/engines/dummy-offline.py+12 searx/engines/genius.py+1 searx/preferences.py+44 searx/query.py+3−1 searx/search.py+4−1 searx/templates/oscar/preferences.html+6 searx/webapp.py+12−5 tests/unit/test_search.py+71−19

searx/__init__.py+1

@@ -38,6 +38,7 @@ def check_settings_yml(file_name):

3838 else:

3939 return None

4040

41+

4142# find location of settings.yml

4243if 'SEARX_SETTINGS_PATH' in environ:

4344 # if possible set path to settings using the

searx/engines/__init__.py+7−2

@@ -54,7 +54,8 @@ engine_default_args = {'paging': False,

5454 'suspend_end_time': 0,

5555 'continuous_errors': 0,

5656 'time_range_support': False,

57− 'offline': False}

57+ 'offline': False,

58+ 'tokens': []}

5859

5960

6061def load_engine(engine_data):

@@ -160,7 +161,7 @@ def to_percentage(stats, maxvalue):

160161 return stats

161162

162163

163−def get_engines_stats():

164+def get_engines_stats(preferences):

164165 # TODO refactor

165166 pageloads = []

166167 engine_times = []

@@ -171,8 +172,12 @@ def get_engines_stats():

171172

172173 max_pageload = max_engine_times = max_results = max_score = max_errors = max_score_per_result = 0 # noqa

173174 for engine in engines.values():

175+ if not preferences.validate_token(engine):

176+ continue

177+

174178 if engine.stats['search_count'] == 0:

175179 continue

180+

176181 results_num = \

177182 engine.stats['result_count'] / float(engine.stats['search_count'])

178183

searx/engines/dummy-offline.pynew file+12

@@ -0,0 +1,12 @@

1+"""

2+ Dummy Offline

4+ @results one result

5+ @stable yes

6+"""

9+def search(query, request_params):

10+ return [{

11+ 'result': 'this is what you get',

12+ }]

searx/engines/genius.py+1

@@ -72,6 +72,7 @@ def parse_album(hit):

7272 result.update({'content': 'Released: {}'.format(year)})

7373 return result

7474

75+

7576parse = {'lyric': parse_lyric, 'song': parse_lyric, 'artist': parse_artist, 'album': parse_album}

7677

7778

searx/preferences.py+44

@@ -104,6 +104,31 @@ class MultipleChoiceSetting(EnumStringSetting):

104104 resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE)

105105

106106

107+class SetSetting(Setting):

108+ def _post_init(self):

109+ if not hasattr(self, 'values'):

110+ self.values = set()

111+

112+ def get_value(self):

113+ return ','.join(self.values)

114+

115+ def parse(self, data):

116+ if data == '':

117+ self.values = set()

118+ return

119+

120+ elements = data.split(',')

121+ for element in elements:

122+ self.values.add(element)

123+

124+ def parse_form(self, data):

125+ elements = data.split(',')

126+ self.values = set(elements)

127+

128+ def save(self, name, resp):

129+ resp.set_cookie(name, ','.join(self.values), max_age=COOKIE_MAX_AGE)

130+

131+

107132class SearchLanguageSetting(EnumStringSetting):

108133 """Available choices may change, so user's value may not be in choices anymore"""

109134

@@ -272,6 +297,7 @@ class Preferences(object):

272297

273298 self.engines = EnginesSetting('engines', choices=engines)

274299 self.plugins = PluginsSetting('plugins', choices=plugins)

300+ self.tokens = SetSetting('tokens')

275301 self.unknown_params = {}

276302

277303 def get_as_url_params(self):

@@ -288,6 +314,8 @@ class Preferences(object):

288314 settings_kv['disabled_plugins'] = ','.join(self.plugins.disabled)

289315 settings_kv['enabled_plugins'] = ','.join(self.plugins.enabled)

290316

317+ settings_kv['tokens'] = ','.join(self.tokens.values)

318+

291319 return urlsafe_b64encode(compress(urlencode(settings_kv).encode('utf-8'))).decode('utf-8')

292320

293321 def parse_encoded_data(self, input_data):

@@ -307,6 +335,8 @@ class Preferences(object):

307335 elif user_setting_name == 'disabled_plugins':

308336 self.plugins.parse_cookie((input_data.get('disabled_plugins', ''),

309337 input_data.get('enabled_plugins', '')))

338+ elif user_setting_name == 'tokens':

339+ self.tokens.parse(user_setting)

310340 elif not any(user_setting_name.startswith(x) for x in [

311341 'enabled_',

312342 'disabled_',

@@ -328,6 +358,8 @@ class Preferences(object):

328358 enabled_categories.append(user_setting_name[len('category_'):])

329359 elif user_setting_name.startswith('plugin_'):

330360 disabled_plugins.append(user_setting_name)

361+ elif user_setting_name == 'tokens':

362+ self.tokens.parse_form(user_setting)

331363 else:

332364 self.unknown_params[user_setting_name] = user_setting

333365 self.key_value_settings['categories'].parse_form(enabled_categories)

@@ -346,6 +378,18 @@ class Preferences(object):

346378 user_setting.save(user_setting_name, resp)

347379 self.engines.save(resp)

348380 self.plugins.save(resp)

381+ self.tokens.save('tokens', resp)

349382 for k, v in self.unknown_params.items():

350383 resp.set_cookie(k, v, max_age=COOKIE_MAX_AGE)

351384 return resp

385+

386+ def validate_token(self, engine):

387+ valid = True

388+ if hasattr(engine, 'tokens') and engine.tokens:

389+ valid = False

390+ for token in self.tokens.values:

391+ if token in engine.tokens:

392+ valid = True

393+ break

394+

395+ return valid

searx/query.py+3−1

@@ -177,7 +177,8 @@ class RawTextQuery(object):

177177class SearchQuery(object):

178178 """container for all the search parameters (query, language, etc...)"""

179179

180− def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range, timeout_limit=None):

180+ def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range,

181+ timeout_limit=None, preferences=None):

181182 self.query = query.encode('utf-8')

182183 self.engines = engines

183184 self.categories = categories

@@ -186,6 +187,7 @@ class SearchQuery(object):

186187 self.pageno = pageno

187188 self.time_range = None if time_range in ('', 'None', None) else time_range

188189 self.timeout_limit = timeout_limit

190+ self.preferences = preferences

189191

190192 def __str__(self):

191193 return str(self.query) + ";" + str(self.engines)

searx/search.py+4−1

@@ -407,7 +407,7 @@ def get_search_query_from_webapp(preferences, form):

407407

408408 return (SearchQuery(query, query_engines, query_categories,

409409 query_lang, query_safesearch, query_pageno,

410− query_time_range, query_timeout),

410+ query_time_range, query_timeout, preferences),

411411 raw_text_query)

412412

413413

@@ -459,6 +459,9 @@ class Search(object):

459459

460460 engine = engines[selected_engine['name']]

461461

462+ if not search_query.preferences.validate_token(engine):

463+ continue

464+

462465 # skip suspended engines

463466 if engine.suspend_end_time >= time():

464467 logger.debug('Engine currently suspended: %s', selected_engine['name'])

searx/templates/oscar/preferences.html+6

@@ -131,6 +131,12 @@

131131 {% endfor %}

132132 </select>

133133 {{ preferences_item_footer(info, label, rtl) }}

134+

135+ {% set label = _('Engine tokens') %}

136+ {% set info = _('Access tokens for private engines') %}

137+ {{ preferences_item_header(info, label, rtl) }}

138+ <input class="form-control" id='tokens' name='tokens' value='{{ preferences.tokens.get_value() }}'/>

139+ {{ preferences_item_footer(info, label, rtl) }}

134140 </div>

135141 </fieldset>

136142 </div>

searx/webapp.py+12−5

@@ -731,8 +731,13 @@ def preferences():

731731 # stats for preferences page

732732 stats = {}

733733

734+ engines_by_category = {}

734735 for c in categories:

736+ engines_by_category[c] = []

735737 for e in categories[c]:

738+ if not request.preferences.validate_token(e):

739+ continue

740+

736741 stats[e.name] = {'time': None,

737742 'warn_timeout': False,

738743 'warn_time': False}

@@ -740,9 +745,11 @@ def preferences():

740745 stats[e.name]['warn_timeout'] = True

741746 stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)

742747

748+ engines_by_category[c].append(e)

749+

743750 # get first element [0], the engine time,

744751 # and then the second element [1] : the time (the first one is the label)

745− for engine_stat in get_engines_stats()[0][1]:

752+ for engine_stat in get_engines_stats(request.preferences)[0][1]:

746753 stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)

747754 if engine_stat.get('avg') > settings['outgoing']['request_timeout']:

748755 stats[engine_stat.get('name')]['warn_time'] = True

@@ -752,7 +759,7 @@ def preferences():

752759 locales=settings['locales'],

753760 current_locale=get_locale(),

754761 image_proxy=image_proxy,

755− engines_by_category=categories,

762+ engines_by_category=engines_by_category,

756763 stats=stats,

757764 answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],

758765 disabled_engines=disabled_engines,

@@ -828,7 +835,7 @@ def image_proxy():

828835@app.route('/stats', methods=['GET'])

829836def stats():

830837 """Render engine statistics page."""

831− stats = get_engines_stats()

838+ stats = get_engines_stats(request.preferences)

832839 return render(

833840 'stats.html',

834841 stats=stats,

@@ -891,7 +898,7 @@ def clear_cookies():

891898@app.route('/config')

892899def config():

893900 return jsonify({'categories': list(categories.keys()),

894− 'engines': [{'name': engine_name,

901+ 'engines': [{'name': name,

895902 'categories': engine.categories,

896903 'shortcut': engine.shortcut,

897904 'enabled': not engine.disabled,

@@ -904,7 +911,7 @@ def config():

904911 'safesearch': engine.safesearch,

905912 'time_range_support': engine.time_range_support,

906913 'timeout': engine.timeout}

907− for engine_name, engine in engines.items()],

914+ for name, engine in engines.items() if request.preferences.validate_token(engine)],

908915 'plugins': [{'name': plugin.name,

909916 'enabled': plugin.default_on}

910917 for plugin in plugins],

tests/unit/test_search.py+71−19

@@ -1,60 +1,112 @@

11# -*- coding: utf-8 -*-

33from searx.testing import SearxTestCase

4+from searx.preferences import Preferences

5+from searx.engines import engines

5−import searx.preferences

67import searx.search

7−import searx.engines

10+SAFESEARCH = 0

11+PAGENO = 1

12+PUBLIC_ENGINE_NAME = 'general dummy'

13+PRIVATE_ENGINE_NAME = 'general private offline'

14+TEST_ENGINES = [

15+ {

16+ 'name': PUBLIC_ENGINE_NAME,

17+ 'engine': 'dummy',

18+ 'categories': 'general',

19+ 'shortcut': 'gd',

20+ 'timeout': 3.0,

21+ 'tokens': [],

22+ },

23+ {

24+ 'name': PRIVATE_ENGINE_NAME,

25+ 'engine': 'dummy-offline',

26+ 'categories': 'general',

27+ 'shortcut': 'do',

28+ 'timeout': 3.0,

29+ 'offline': True,

30+ 'tokens': ['my-token'],

31+ },

32+]

833

934

1035class SearchTestCase(SearxTestCase):

1136

1237 @classmethod

1338 def setUpClass(cls):

14− searx.engines.initialize_engines([{

15− 'name': 'general dummy',

16− 'engine': 'dummy',

17− 'categories': 'general',

18− 'shortcut': 'gd',

19− 'timeout': 3.0

20− }])

39+ searx.engines.initialize_engines(TEST_ENGINES)

2140

2241 def test_timeout_simple(self):

2342 searx.search.max_request_timeout = None

24− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

25− ['general'], 'en-US', 0, 1, None, None)

43+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

44+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

45+ preferences=Preferences(['oscar'], ['general'], engines, []))

2646 search = searx.search.Search(search_query)

2747 search.search()

2848 self.assertEquals(search.actual_timeout, 3.0)

2949

3050 def test_timeout_query_above_default_nomax(self):

3151 searx.search.max_request_timeout = None

32− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

33− ['general'], 'en-US', 0, 1, None, 5.0)

52+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

53+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

54+ preferences=Preferences(['oscar'], ['general'], engines, []))

3455 search = searx.search.Search(search_query)

3556 search.search()

3657 self.assertEquals(search.actual_timeout, 3.0)

3758

3859 def test_timeout_query_below_default_nomax(self):

3960 searx.search.max_request_timeout = None

40− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

41− ['general'], 'en-US', 0, 1, None, 1.0)

61+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

62+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0,

63+ preferences=Preferences(['oscar'], ['general'], engines, []))

4264 search = searx.search.Search(search_query)

4365 search.search()

4466 self.assertEquals(search.actual_timeout, 1.0)

4567

4668 def test_timeout_query_below_max(self):

4769 searx.search.max_request_timeout = 10.0

48− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

49− ['general'], 'en-US', 0, 1, None, 5.0)

70+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

71+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

72+ preferences=Preferences(['oscar'], ['general'], engines, []))

5073 search = searx.search.Search(search_query)

5174 search.search()

5275 self.assertEquals(search.actual_timeout, 5.0)

5376

5477 def test_timeout_query_above_max(self):

5578 searx.search.max_request_timeout = 10.0

56− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

57− ['general'], 'en-US', 0, 1, None, 15.0)

79+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

80+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0,

81+ preferences=Preferences(['oscar'], ['general'], engines, []))

5882 search = searx.search.Search(search_query)

5983 search.search()

6084 self.assertEquals(search.actual_timeout, 10.0)

85+

86+ def test_query_private_engine_without_token(self):

87+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

88+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

89+ preferences=Preferences(['oscar'], ['general'], engines, []))

90+ search = searx.search.Search(search_query)

91+ results = search.search()

92+ self.assertEquals(results.results_length(), 0)

93+

94+ def test_query_private_engine_with_incorrect_token(self):

95+ preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

96+ preferences_with_tokens.parse_dict({'tokens': 'bad-token'})

97+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

98+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

99+ preferences=preferences_with_tokens)

100+ search = searx.search.Search(search_query)

101+ results = search.search()

102+ self.assertEquals(results.results_length(), 0)

103+

104+ def test_query_private_engine_with_correct_token(self):

105+ preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

106+ preferences_with_tokens.parse_dict({'tokens': 'my-token'})

107+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

108+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

109+ preferences=preferences_with_tokens)

110+ search = searx.search.Search(search_query)

111+ results = search.search()

112+ self.assertEquals(results.results_length(), 1)