691d127 · searx · git.iko.soy

git.iko.soy/searx

[mod] check the engine tokens in searx/webadapter.py instead of searx/search.py

commit691d12726be1411d8c80895e927293bdcc3e9572 authorAlexandre Flament <alex@al-f.net> dateTue 22 Sep 2020, 16:55 (6 years ago) parenteecfff2

6 files changed+95−64

compare with master

unifiedsplit

searx/search.py+1−5 searx/webadapter.py+31−2 searx/webapp.py+1−1 tests/unit/test_search.py+7−55 tests/unit/test_webadapter.py+54 utils/standalone_searx.py+1−1

searx/search.py+1−5

@@ -62,7 +62,7 @@ class SearchQuery:

6262 """container for all the search parameters (query, language, etc...)"""

6363

6464 def __init__(self, query, engineref_list, categories, lang, safesearch, pageno, time_range,

65− timeout_limit=None, preferences=None, external_bang=None):

65+ timeout_limit=None, external_bang=None):

6666 self.query = query

6767 self.engineref_list = engineref_list

6868 self.categories = categories

@@ -71,7 +71,6 @@ class SearchQuery:

7171 self.pageno = pageno

7272 self.time_range = time_range

7373 self.timeout_limit = timeout_limit

74− self.preferences = preferences

7574 self.external_bang = external_bang

7675

7776 def __str__(self):

@@ -311,9 +310,6 @@ class Search:

311310 return False

312311

313312 def _is_accepted(self, engine_name, engine):

314− if not self.search_query.preferences.validate_token(engine):

315− return False

316−

317313 # skip suspended engines

318314 if engine.suspend_end_time >= time():

319315 logger.debug('Engine currently suspended: %s', engine_name)

searx/webadapter.py+31−2

@@ -11,6 +11,31 @@ def deduplicate_engineref_list(engineref_list):

1111 return engineref_dict.values()

1212

1313

14+def validate_engineref_list(engineref_list, preferences):

15+ """

16+ Validate query_engines according to the preferences

17+ Returns:

18+ list of existing engines with a validated token

19+ list of unknown engine

20+ list of engine with invalid token according to the preferences

21+ """

22+ valid = []

23+ unknown = []

24+ no_token = []

25+ for engineref in engineref_list:

26+ if engineref.name not in engines:

27+ unknown.append(engineref)

28+ continue

29+

30+ engine = engines[engineref.name]

31+ if not preferences.validate_token(engine):

32+ no_token.append(engineref)

33+ continue

34+

35+ valid.append(engineref)

36+ return valid, unknown, no_token

37+

38+

1439def get_search_query_from_webapp(preferences, form):

1540 # no text for the query ?

1641 if not form.get('q'):

@@ -152,10 +177,14 @@ def get_search_query_from_webapp(preferences, form):

152177 if (engine.name, categ) not in disabled_engines)

153178

154179 query_engineref_list = deduplicate_engineref_list(query_engineref_list)

180+ query_engineref_list, query_engineref_list_unknown, query_engineref_list_notoken =\

181+ validate_engineref_list(query_engineref_list, preferences)

155182 external_bang = raw_text_query.external_bang

156183

157184 return (SearchQuery(query, query_engineref_list, query_categories,

158185 query_lang, query_safesearch, query_pageno,

159− query_time_range, query_timeout, preferences,

186+ query_time_range, query_timeout,

160187 external_bang=external_bang),

161− raw_text_query)

188+ raw_text_query,

189+ query_engineref_list_unknown,

190+ query_engineref_list_notoken)

searx/webapp.py+1−1

@@ -562,7 +562,7 @@ def index():

562562 raw_text_query = None

563563 result_container = None

564564 try:

565− search_query, raw_text_query = get_search_query_from_webapp(request.preferences, request.form)

565+ search_query, raw_text_query, _, _ = get_search_query_from_webapp(request.preferences, request.form)

566566 # search = Search(search_query) # without plugins

567567 search = SearchWithPlugins(search_query, request.user_plugins, request)

568568

tests/unit/test_search.py+7−55

@@ -1,17 +1,13 @@

11# -*- coding: utf-8 -*-

33from searx.testing import SearxTestCase

4−from searx.preferences import Preferences

5−from searx.engines import engines

6−

7−import searx.search

84from searx.search import SearchQuery, EngineRef

5+import searx.search

107

118SAFESEARCH = 0

129PAGENO = 1

1310PUBLIC_ENGINE_NAME = 'general dummy'

14−PRIVATE_ENGINE_NAME = 'general private offline'

1511TEST_ENGINES = [

1612 {

1713 'name': PUBLIC_ENGINE_NAME,

@@ -21,15 +17,6 @@ TEST_ENGINES = [

2117 'timeout': 3.0,

2218 'tokens': [],

2319 },

24− {

25− 'name': PRIVATE_ENGINE_NAME,

26− 'engine': 'dummy-offline',

27− 'categories': 'general',

28− 'shortcut': 'do',

29− 'timeout': 3.0,

30− 'offline': True,

31− 'tokens': ['my-token'],

32− },

3320]

3421

3522

@@ -42,8 +29,7 @@ class SearchTestCase(SearxTestCase):

4229 def test_timeout_simple(self):

4330 searx.search.max_request_timeout = None

4431 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

45− ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

46− preferences=Preferences(['oscar'], ['general'], engines, []))

32+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None)

4733 search = searx.search.Search(search_query)

4834 search.search()

4935 self.assertEqual(search.actual_timeout, 3.0)

@@ -51,8 +37,7 @@ class SearchTestCase(SearxTestCase):

5137 def test_timeout_query_above_default_nomax(self):

5238 searx.search.max_request_timeout = None

5339 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

54− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

55− preferences=Preferences(['oscar'], ['general'], engines, []))

40+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0)

5641 search = searx.search.Search(search_query)

5742 search.search()

5843 self.assertEqual(search.actual_timeout, 3.0)

@@ -60,8 +45,7 @@ class SearchTestCase(SearxTestCase):

6045 def test_timeout_query_below_default_nomax(self):

6146 searx.search.max_request_timeout = None

6247 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

63− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0,

64− preferences=Preferences(['oscar'], ['general'], engines, []))

48+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0)

6549 search = searx.search.Search(search_query)

6650 search.search()

6751 self.assertEqual(search.actual_timeout, 1.0)

@@ -69,8 +53,7 @@ class SearchTestCase(SearxTestCase):

6953 def test_timeout_query_below_max(self):

7054 searx.search.max_request_timeout = 10.0

7155 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

72− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

73− preferences=Preferences(['oscar'], ['general'], engines, []))

56+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0)

7457 search = searx.search.Search(search_query)

7558 search.search()

7659 self.assertEqual(search.actual_timeout, 5.0)

@@ -78,45 +61,15 @@ class SearchTestCase(SearxTestCase):

7861 def test_timeout_query_above_max(self):

7962 searx.search.max_request_timeout = 10.0

8063 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

81− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0,

82− preferences=Preferences(['oscar'], ['general'], engines, []))

64+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0)

8365 search = searx.search.Search(search_query)

8466 search.search()

8567 self.assertEqual(search.actual_timeout, 10.0)

8668

87− def test_query_private_engine_without_token(self):

88− search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

89− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

90− preferences=Preferences(['oscar'], ['general'], engines, []))

91− search = searx.search.Search(search_query)

92− results = search.search()

93− self.assertEqual(results.results_length(), 0)

94−

95− def test_query_private_engine_with_incorrect_token(self):

96− preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

97− preferences_with_tokens.parse_dict({'tokens': 'bad-token'})

98− search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

99− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

100− preferences=preferences_with_tokens)

101− search = searx.search.Search(search_query)

102− results = search.search()

103− self.assertEqual(results.results_length(), 0)

104−

105− def test_query_private_engine_with_correct_token(self):

106− preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

107− preferences_with_tokens.parse_dict({'tokens': 'my-token'})

108− search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

109− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

110− preferences=preferences_with_tokens)

111− search = searx.search.Search(search_query)

112− results = search.search()

113− self.assertEqual(results.results_length(), 1)

114−

11569 def test_external_bang(self):

11670 search_query = SearchQuery('yes yes',

11771 [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

11872 ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

119− preferences=Preferences(['oscar'], ['general'], engines, [],),

12073 external_bang="yt")

12174 search = searx.search.Search(search_query)

12275 results = search.search()

@@ -125,8 +78,7 @@ class SearchTestCase(SearxTestCase):

12578

12679 search_query = SearchQuery('youtube never gonna give you up',

12780 [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

128− ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

129− preferences=Preferences(['oscar'], ['general'], engines, []),)

81+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None)

13082

13183 search = searx.search.Search(search_query)

13284 results = search.search()

tests/unit/test_webadapter.pynew file+54

@@ -0,0 +1,54 @@

1+# -*- coding: utf-8 -*-

3+from searx.testing import SearxTestCase

4+from searx.preferences import Preferences

5+from searx.engines import engines

7+import searx.search

8+from searx.search import EngineRef, SearchQuery

9+from searx.webadapter import validate_engineref_list

10+

11+

12+PRIVATE_ENGINE_NAME = 'general private offline'

13+TEST_ENGINES = [

14+ {

15+ 'name': PRIVATE_ENGINE_NAME,

16+ 'engine': 'dummy-offline',

17+ 'categories': 'general',

18+ 'shortcut': 'do',

19+ 'timeout': 3.0,

20+ 'offline': True,

21+ 'tokens': ['my-token'],

22+ },

23+]

24+SEARCHQUERY = [EngineRef(PRIVATE_ENGINE_NAME, 'general')]

25+

26+

27+class ValidateQueryCase(SearxTestCase):

28+

29+ @classmethod

30+ def setUpClass(cls):

31+ searx.engines.initialize_engines(TEST_ENGINES)

32+

33+ def test_query_private_engine_without_token(self):

34+ preferences = Preferences(['oscar'], ['general'], engines, [])

35+ valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences)

36+ self.assertEqual(len(valid), 0)

37+ self.assertEqual(len(unknown), 0)

38+ self.assertEqual(len(invalid_token), 1)

39+

40+ def test_query_private_engine_with_incorrect_token(self):

41+ preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

42+ preferences_with_tokens.parse_dict({'tokens': 'bad-token'})

43+ valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences_with_tokens)

44+ self.assertEqual(len(valid), 0)

45+ self.assertEqual(len(unknown), 0)

46+ self.assertEqual(len(invalid_token), 1)

47+

48+ def test_query_private_engine_with_correct_token(self):

49+ preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

50+ preferences_with_tokens.parse_dict({'tokens': 'my-token'})

51+ valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences_with_tokens)

52+ self.assertEqual(len(valid), 1)

53+ self.assertEqual(len(unknown), 0)

54+ self.assertEqual(len(invalid_token), 0)

utils/standalone_searx.py+1−1

@@ -65,7 +65,7 @@ form = {

6565preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, [])

6666preferences.key_value_settings['safesearch'].parse(args.safesearch)

6767

68−search_query, raw_text_query = searx.webadapter.get_search_query_from_webapp(preferences, form)

68+search_query, raw_text_query, _, _ = searx.webadapter.get_search_query_from_webapp(preferences, form)

6969search = searx.search.Search(search_query)

7070result_container = search.search()

7171