691d127 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[mod] check the engine tokens in searx/webadapter.py instead of searx/search.py

691d127Alexandre Flament6 years ago

searx/search.py+1−5

compare with master

unifiedsplit

@@ -62,7 +62,7 @@ class SearchQuery:

62 """container for all the search parameters (query, language, etc...)"""

64 def __init__(self, query, engineref_list, categories, lang, safesearch, pageno, time_range,

65− timeout_limit=None, preferences=None, external_bang=None):

65+ timeout_limit=None, external_bang=None):

66 self.query = query

67 self.engineref_list = engineref_list

68 self.categories = categories

@@ -71,7 +71,6 @@ class SearchQuery:

71 self.pageno = pageno

72 self.time_range = time_range

73 self.timeout_limit = timeout_limit

74− self.preferences = preferences

75 self.external_bang = external_bang

74 self.external_bang = external_bang

77 def __str__(self):

76 def __str__(self):

@@ -311,9 +310,6 @@ class Search:

311 return False

310 return False

312

311

313 def _is_accepted(self, engine_name, engine):

312 def _is_accepted(self, engine_name, engine):

314− if not self.search_query.preferences.validate_token(engine):

315− return False

316−

317 # skip suspended engines

313 # skip suspended engines

318 if engine.suspend_end_time >= time():

314 if engine.suspend_end_time >= time():

319 logger.debug('Engine currently suspended: %s', engine_name)

315 logger.debug('Engine currently suspended: %s', engine_name)

searx/webadapter.py+31−2

@@ -11,6 +11,31 @@ def deduplicate_engineref_list(engineref_list):

11 return engineref_dict.values()

14+def validate_engineref_list(engineref_list, preferences):

15+ """

16+ Validate query_engines according to the preferences

17+ Returns:

18+ list of existing engines with a validated token

19+ list of unknown engine

20+ list of engine with invalid token according to the preferences

21+ """

22+ valid = []

23+ unknown = []

24+ no_token = []

25+ for engineref in engineref_list:

26+ if engineref.name not in engines:

27+ unknown.append(engineref)

28+ continue

29+

30+ engine = engines[engineref.name]

31+ if not preferences.validate_token(engine):

32+ no_token.append(engineref)

33+ continue

34+

35+ valid.append(engineref)

36+ return valid, unknown, no_token

37+

38+

14def get_search_query_from_webapp(preferences, form):

39def get_search_query_from_webapp(preferences, form):

15 # no text for the query ?

40 # no text for the query ?

16 if not form.get('q'):

41 if not form.get('q'):

@@ -152,10 +177,14 @@ def get_search_query_from_webapp(preferences, form):

152 if (engine.name, categ) not in disabled_engines)

177 if (engine.name, categ) not in disabled_engines)

153

178

154 query_engineref_list = deduplicate_engineref_list(query_engineref_list)

179 query_engineref_list = deduplicate_engineref_list(query_engineref_list)

180+ query_engineref_list, query_engineref_list_unknown, query_engineref_list_notoken =\

181+ validate_engineref_list(query_engineref_list, preferences)

155 external_bang = raw_text_query.external_bang

182 external_bang = raw_text_query.external_bang

156

183

157 return (SearchQuery(query, query_engineref_list, query_categories,

184 return (SearchQuery(query, query_engineref_list, query_categories,

158 query_lang, query_safesearch, query_pageno,

185 query_lang, query_safesearch, query_pageno,

159− query_time_range, query_timeout, preferences,

186+ query_time_range, query_timeout,

160 external_bang=external_bang),

187 external_bang=external_bang),

161− raw_text_query)

188+ raw_text_query,

189+ query_engineref_list_unknown,

190+ query_engineref_list_notoken)

searx/webapp.py+1−1

@@ -562,7 +562,7 @@ def index():

562 raw_text_query = None

563 result_container = None

564 try:

565− search_query, raw_text_query = get_search_query_from_webapp(request.preferences, request.form)

565+ search_query, raw_text_query, _, _ = get_search_query_from_webapp(request.preferences, request.form)

566 # search = Search(search_query) # without plugins

567 search = SearchWithPlugins(search_query, request.user_plugins, request)

568

tests/unit/test_search.py+7−55

@@ -1,17 +1,13 @@

1# -*- coding: utf-8 -*-

3from searx.testing import SearxTestCase

4−from searx.preferences import Preferences

5−from searx.engines import engines

6−

7−import searx.search

8from searx.search import SearchQuery, EngineRef

4from searx.search import SearchQuery, EngineRef

5+import searx.search

11SAFESEARCH = 0

8SAFESEARCH = 0

12PAGENO = 1

9PAGENO = 1

13PUBLIC_ENGINE_NAME = 'general dummy'

10PUBLIC_ENGINE_NAME = 'general dummy'

14−PRIVATE_ENGINE_NAME = 'general private offline'

15TEST_ENGINES = [

11TEST_ENGINES = [

16 {

12 {

17 'name': PUBLIC_ENGINE_NAME,

13 'name': PUBLIC_ENGINE_NAME,

@@ -21,15 +17,6 @@ TEST_ENGINES = [

21 'timeout': 3.0,

17 'timeout': 3.0,

22 'tokens': [],

18 'tokens': [],

23 },

19 },

24− {

25− 'name': PRIVATE_ENGINE_NAME,

26− 'engine': 'dummy-offline',

27− 'categories': 'general',

28− 'shortcut': 'do',

29− 'timeout': 3.0,

30− 'offline': True,

31− 'tokens': ['my-token'],

32− },

33]

20]

@@ -42,8 +29,7 @@ class SearchTestCase(SearxTestCase):

42 def test_timeout_simple(self):

29 def test_timeout_simple(self):

43 searx.search.max_request_timeout = None

30 searx.search.max_request_timeout = None

44 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

31 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

45− ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

32+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None)

46− preferences=Preferences(['oscar'], ['general'], engines, []))

47 search = searx.search.Search(search_query)

33 search = searx.search.Search(search_query)

48 search.search()

34 search.search()

49 self.assertEqual(search.actual_timeout, 3.0)

35 self.assertEqual(search.actual_timeout, 3.0)

@@ -51,8 +37,7 @@ class SearchTestCase(SearxTestCase):

51 def test_timeout_query_above_default_nomax(self):

37 def test_timeout_query_above_default_nomax(self):

52 searx.search.max_request_timeout = None

38 searx.search.max_request_timeout = None

53 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

39 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

54− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

40+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0)

55− preferences=Preferences(['oscar'], ['general'], engines, []))

56 search = searx.search.Search(search_query)

41 search = searx.search.Search(search_query)

57 search.search()

42 search.search()

58 self.assertEqual(search.actual_timeout, 3.0)

43 self.assertEqual(search.actual_timeout, 3.0)

@@ -60,8 +45,7 @@ class SearchTestCase(SearxTestCase):

60 def test_timeout_query_below_default_nomax(self):

45 def test_timeout_query_below_default_nomax(self):

61 searx.search.max_request_timeout = None

46 searx.search.max_request_timeout = None

62 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

47 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

63− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0,

48+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0)

64− preferences=Preferences(['oscar'], ['general'], engines, []))

65 search = searx.search.Search(search_query)

49 search = searx.search.Search(search_query)

66 search.search()

50 search.search()

67 self.assertEqual(search.actual_timeout, 1.0)

51 self.assertEqual(search.actual_timeout, 1.0)

@@ -69,8 +53,7 @@ class SearchTestCase(SearxTestCase):

69 def test_timeout_query_below_max(self):

53 def test_timeout_query_below_max(self):

70 searx.search.max_request_timeout = 10.0

54 searx.search.max_request_timeout = 10.0

71 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

55 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

72− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

56+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0)

73− preferences=Preferences(['oscar'], ['general'], engines, []))

74 search = searx.search.Search(search_query)

57 search = searx.search.Search(search_query)

75 search.search()

58 search.search()

76 self.assertEqual(search.actual_timeout, 5.0)

59 self.assertEqual(search.actual_timeout, 5.0)

@@ -78,45 +61,15 @@ class SearchTestCase(SearxTestCase):

78 def test_timeout_query_above_max(self):

61 def test_timeout_query_above_max(self):

79 searx.search.max_request_timeout = 10.0

62 searx.search.max_request_timeout = 10.0

80 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

63 search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

81− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0,

64+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0)

82− preferences=Preferences(['oscar'], ['general'], engines, []))

83 search = searx.search.Search(search_query)

65 search = searx.search.Search(search_query)

84 search.search()

66 search.search()

85 self.assertEqual(search.actual_timeout, 10.0)

67 self.assertEqual(search.actual_timeout, 10.0)

87− def test_query_private_engine_without_token(self):

88− search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

89− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

90− preferences=Preferences(['oscar'], ['general'], engines, []))

91− search = searx.search.Search(search_query)

92− results = search.search()

93− self.assertEqual(results.results_length(), 0)

94−

95− def test_query_private_engine_with_incorrect_token(self):

96− preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

97− preferences_with_tokens.parse_dict({'tokens': 'bad-token'})

98− search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

99− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

100− preferences=preferences_with_tokens)

101− search = searx.search.Search(search_query)

102− results = search.search()

103− self.assertEqual(results.results_length(), 0)

104−

105− def test_query_private_engine_with_correct_token(self):

106− preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

107− preferences_with_tokens.parse_dict({'tokens': 'my-token'})

108− search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

109− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

110− preferences=preferences_with_tokens)

111− search = searx.search.Search(search_query)

112− results = search.search()

113− self.assertEqual(results.results_length(), 1)

114−

115 def test_external_bang(self):

69 def test_external_bang(self):

116 search_query = SearchQuery('yes yes',

70 search_query = SearchQuery('yes yes',

117 [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

71 [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

118 ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

72 ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

119− preferences=Preferences(['oscar'], ['general'], engines, [],),

120 external_bang="yt")

73 external_bang="yt")

121 search = searx.search.Search(search_query)

74 search = searx.search.Search(search_query)

122 results = search.search()

75 results = search.search()

@@ -125,8 +78,7 @@ class SearchTestCase(SearxTestCase):

125

126 search_query = SearchQuery('youtube never gonna give you up',

79 search_query = SearchQuery('youtube never gonna give you up',

127 [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

80 [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

128− ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

81+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None)

129− preferences=Preferences(['oscar'], ['general'], engines, []),)

130

131 search = searx.search.Search(search_query)

83 search = searx.search.Search(search_query)

132 results = search.search()

84 results = search.search()

tests/unit/test_webadapter.pynew file+54

@@ -0,0 +1,54 @@

1+# -*- coding: utf-8 -*-

3+from searx.testing import SearxTestCase

4+from searx.preferences import Preferences

5+from searx.engines import engines

7+import searx.search

8+from searx.search import EngineRef, SearchQuery

9+from searx.webadapter import validate_engineref_list

10+

11+

12+PRIVATE_ENGINE_NAME = 'general private offline'

13+TEST_ENGINES = [

14+ {

15+ 'name': PRIVATE_ENGINE_NAME,

16+ 'engine': 'dummy-offline',

17+ 'categories': 'general',

18+ 'shortcut': 'do',

19+ 'timeout': 3.0,

20+ 'offline': True,

21+ 'tokens': ['my-token'],

22+ },

23+]

24+SEARCHQUERY = [EngineRef(PRIVATE_ENGINE_NAME, 'general')]

25+

26+

27+class ValidateQueryCase(SearxTestCase):

28+

29+ @classmethod

30+ def setUpClass(cls):

31+ searx.engines.initialize_engines(TEST_ENGINES)

32+

33+ def test_query_private_engine_without_token(self):

34+ preferences = Preferences(['oscar'], ['general'], engines, [])

35+ valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences)

36+ self.assertEqual(len(valid), 0)

37+ self.assertEqual(len(unknown), 0)

38+ self.assertEqual(len(invalid_token), 1)

39+

40+ def test_query_private_engine_with_incorrect_token(self):

41+ preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

42+ preferences_with_tokens.parse_dict({'tokens': 'bad-token'})

43+ valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences_with_tokens)

44+ self.assertEqual(len(valid), 0)

45+ self.assertEqual(len(unknown), 0)

46+ self.assertEqual(len(invalid_token), 1)

47+

48+ def test_query_private_engine_with_correct_token(self):

49+ preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

50+ preferences_with_tokens.parse_dict({'tokens': 'my-token'})

51+ valid, unknown, invalid_token = validate_engineref_list(SEARCHQUERY, preferences_with_tokens)

52+ self.assertEqual(len(valid), 1)

53+ self.assertEqual(len(unknown), 0)

54+ self.assertEqual(len(invalid_token), 0)

utils/standalone_searx.py+1−1

@@ -65,7 +65,7 @@ form = {

65preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, [])

66preferences.key_value_settings['safesearch'].parse(args.safesearch)

68−search_query, raw_text_query = searx.webadapter.get_search_query_from_webapp(preferences, form)

68+search_query, raw_text_query, _, _ = searx.webadapter.get_search_query_from_webapp(preferences, form)

69search = searx.search.Search(search_query)

70result_container = search.search()