2929495 · searx · git.iko.soy

git.iko.soy/searx

[mod] add searx.search.EngineRef

was previously a Dict with two or three keys: name, category, from_bang make clear that this is a engine reference (see tests/unit/test_search.py for example) all variables using this class are renamed accordingly.

commit2929495112e9869af24986b27066163572bc558a authorAlexandre Flament <alex@al-f.net> dateTue 22 Sep 2020, 16:22 (6 years ago) parent2dbc0de

4 files changed+58−57

compare with master

unifiedsplit

searx/query.py+8−14 searx/search.py+22−13 searx/webadapter.py+17−19 tests/unit/test_search.py+11−11

searx/query.py+8−14

@@ -20,9 +20,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

2020import re

2121

2222from searx.languages import language_codes

23−from searx.engines import (

24− categories, engines, engine_shortcuts

25−)

23+from searx.engines import categories, engines, engine_shortcuts

24+from searx.search import EngineRef

2625

2726

2827VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')

@@ -40,7 +39,7 @@ class RawTextQuery:

4039 self.disabled_engines = disabled_engines

4140

4241 self.query_parts = []

43− self.engines = []

42+ self.enginerefs = []

4443 self.languages = []

4544 self.timeout_limit = None

4645 self.external_bang = None

@@ -135,26 +134,21 @@ class RawTextQuery:

135134 parse_next = True

136135 engine_name = engine_shortcuts[prefix]

137136 if engine_name in engines:

138− self.engines.append({'category': 'none',

139− 'name': engine_name,

140− 'from_bang': True})

137+ self.enginerefs.append(EngineRef(engine_name, 'none', True))

141138

142139 # check if prefix is equal with engine name

143140 elif prefix in engines:

144141 parse_next = True

145− self.engines.append({'category': 'none',

146− 'name': prefix,

147− 'from_bang': True})

142+ self.enginerefs.append(EngineRef(prefix, 'none', True))

148143

149144 # check if prefix is equal with categorie name

150145 elif prefix in categories:

151146 # using all engines for that search, which

152147 # are declared under that categorie name

153148 parse_next = True

154− self.engines.extend({'category': prefix,

155− 'name': engine.name}

156− for engine in categories[prefix]

157− if (engine.name, prefix) not in self.disabled_engines)

149+ self.enginerefs.extend(EngineRef(engine.name, prefix)

150+ for engine in categories[prefix]

151+ if (engine.name, prefix) not in self.disabled_engines)

158152

159153 if query_part[0] == '!':

160154 self.specific = True

searx/search.py+22−13

@@ -47,13 +47,24 @@ else:

4747 exit(1)

4848

4949

50+class EngineRef:

51+

52+ def __init__(self, name, category, from_bang=False):

53+ self.name = name

54+ self.category = category

55+ self.from_bang = from_bang

56+

57+ def __str__(self):

58+ return "(" + self.name + "," + self.category + "," + str(self.from_bang) + ")"

59+

60+

5061class SearchQuery:

5162 """container for all the search parameters (query, language, etc...)"""

5263

53− def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range,

64+ def __init__(self, query, engineref_list, categories, lang, safesearch, pageno, time_range,

5465 timeout_limit=None, preferences=None, external_bang=None):

5566 self.query = query

56− self.engines = engines

67+ self.engineref_list = engineref_list

5768 self.categories = categories

5869 self.lang = lang

5970 self.safesearch = safesearch

@@ -64,7 +75,7 @@ class SearchQuery:

6475 self.external_bang = external_bang

6576

6677 def __str__(self):

67− return self.query + ";" + str(self.engines)

78+ return self.query + ";" + str(self.engineref_list)

6879

6980

7081def send_http_request(engine, request_params):

@@ -318,13 +329,13 @@ class Search:

318329

319330 return True

320331

321− def _get_params(self, selected_engine, user_agent):

322− if selected_engine['name'] not in engines:

332+ def _get_params(self, engineref, user_agent):

333+ if engineref.name not in engines:

323334 return None, None

324335

325− engine = engines[selected_engine['name']]

336+ engine = engines[engineref.name]

326337

327− if not self._is_accepted(selected_engine['name'], engine):

338+ if not self._is_accepted(engineref.name, engine):

328339 return None, None

329340

330341 # set default request parameters

@@ -341,15 +352,13 @@ class Search:

341352 request_params['safesearch'] = self.search_query.safesearch

342353 request_params['time_range'] = self.search_query.time_range

343354

344− request_params['category'] = selected_engine['category']

355+ request_params['category'] = engineref.category

345356 request_params['pageno'] = self.search_query.pageno

346357

347358 return request_params, engine.timeout

348359

349360 # do search-request

350361 def _get_requests(self):

351− global number_of_searches

352−

353362 # init vars

354363 requests = []

355364

@@ -361,14 +370,14 @@ class Search:

361370 default_timeout = 0

362371

363372 # start search-reqest for all selected engines

364− for selected_engine in self.search_query.engines:

373+ for engineref in self.search_query.engineref_list:

365374 # set default request parameters

366− request_params, engine_timeout = self._get_params(selected_engine, user_agent)

375+ request_params, engine_timeout = self._get_params(engineref, user_agent)

367376 if request_params is None:

368377 continue

369378

370379 # append request to list

371− requests.append((selected_engine['name'], self.search_query.query, request_params))

380+ requests.append((engineref.name, self.search_query.query, request_params))

372381

373382 # update default_timeout

374383 default_timeout = max(default_timeout, engine_timeout)

searx/webadapter.py+17−19

@@ -1,14 +1,14 @@

11from searx.exceptions import SearxParameterException

22from searx.query import RawTextQuery, VALID_LANGUAGE_CODE

33from searx.engines import categories, engines

4−from searx.search import SearchQuery

4+from searx.search import SearchQuery, EngineRef

77# remove duplicate queries.

88# FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music'

9−def deduplicate_query_engines(query_engines):

10− uniq_query_engines = {q["category"] + '|' + q["name"]: q for q in query_engines}

11− return uniq_query_engines.values()

9+def deduplicate_engineref_list(engineref_list):

10+ engineref_dict = {q.category + '|' + q.name: q for q in engineref_list}

11+ return engineref_dict.values()

1212

1313

1414def get_search_query_from_webapp(preferences, form):

@@ -68,7 +68,7 @@ def get_search_query_from_webapp(preferences, form):

6868 raise SearxParameterException('time_range', query_time_range)

6969

7070 # query_engines

71− query_engines = raw_text_query.engines

71+ query_engineref_list = raw_text_query.enginerefs

7272

7373 # timeout_limit

7474 query_timeout = raw_text_query.timeout_limit

@@ -87,13 +87,13 @@ def get_search_query_from_webapp(preferences, form):

8787

8888 # if engines are calculated from query,

8989 # set categories by using that informations

90− if query_engines and raw_text_query.specific:

90+ if query_engineref_list and raw_text_query.specific:

9191 additional_categories = set()

92− for engine in query_engines:

93− if 'from_bang' in engine and engine['from_bang']:

92+ for engineref in query_engineref_list:

93+ if engineref.from_bang:

9494 additional_categories.add('none')

9595 else:

96− additional_categories.add(engine['category'])

96+ additional_categories.add(engineref.category)

9797 query_categories = list(additional_categories)

9898

9999 # otherwise, using defined categories to

@@ -105,11 +105,10 @@ def get_search_query_from_webapp(preferences, form):

105105 if pd_name == 'categories':

106106 query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories)

107107 elif pd_name == 'engines':

108− pd_engines = [{'category': engines[engine].categories[0],

109− 'name': engine}

108+ pd_engines = [EngineRef(engineref, engines[engineref].categories[0])

110109 for engine in map(str.strip, pd.split(',')) if engine in engines]

111110 if pd_engines:

112− query_engines.extend(pd_engines)

111+ query_engineref_list.extend(pd_engines)

113112 load_default_categories = False

114113 elif pd_name.startswith('category_'):

115114 category = pd_name[9:]

@@ -128,7 +127,7 @@ def get_search_query_from_webapp(preferences, form):

128127 if not load_default_categories:

129128 if not query_categories:

130129 query_categories = list(set(engine['category']

131− for engine in query_engines))

130+ for engine in query_engineref_list))

132131 else:

133132 # if no category is specified for this search,

134133 # using user-defined default-configuration which

@@ -147,15 +146,14 @@ def get_search_query_from_webapp(preferences, form):

147146 # using all engines for that search, which are

148147 # declared under the specific categories

149148 for categ in query_categories:

150− query_engines.extend({'category': categ,

151− 'name': engine.name}

152− for engine in categories[categ]

153− if (engine.name, categ) not in disabled_engines)

149+ query_engineref_list.extend(EngineRef(engine.name, categ)

150+ for engine in categories[categ]

151+ if (engine.name, categ) not in disabled_engines)

154152

155− query_engines = deduplicate_query_engines(query_engines)

153+ query_engineref_list = deduplicate_engineref_list(query_engineref_list)

156154 external_bang = raw_text_query.external_bang

157155

158− return (SearchQuery(query, query_engines, query_categories,

156+ return (SearchQuery(query, query_engineref_list, query_categories,

159157 query_lang, query_safesearch, query_pageno,

160158 query_time_range, query_timeout, preferences,

161159 external_bang=external_bang),

tests/unit/test_search.py+11−11

@@ -5,7 +5,7 @@ from searx.preferences import Preferences

55from searx.engines import engines

77import searx.search

8−from searx.search import SearchQuery

8+from searx.search import SearchQuery, EngineRef

1010

1111SAFESEARCH = 0

@@ -41,7 +41,7 @@ class SearchTestCase(SearxTestCase):

4141

4242 def test_timeout_simple(self):

4343 searx.search.max_request_timeout = None

44− search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

44+ search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

4545 ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

4646 preferences=Preferences(['oscar'], ['general'], engines, []))

4747 search = searx.search.Search(search_query)

@@ -50,7 +50,7 @@ class SearchTestCase(SearxTestCase):

5050

5151 def test_timeout_query_above_default_nomax(self):

5252 searx.search.max_request_timeout = None

53− search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

53+ search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

5454 ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

5555 preferences=Preferences(['oscar'], ['general'], engines, []))

5656 search = searx.search.Search(search_query)

@@ -59,7 +59,7 @@ class SearchTestCase(SearxTestCase):

5959

6060 def test_timeout_query_below_default_nomax(self):

6161 searx.search.max_request_timeout = None

62− search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

62+ search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

6363 ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0,

6464 preferences=Preferences(['oscar'], ['general'], engines, []))

6565 search = searx.search.Search(search_query)

@@ -68,7 +68,7 @@ class SearchTestCase(SearxTestCase):

6868

6969 def test_timeout_query_below_max(self):

7070 searx.search.max_request_timeout = 10.0

71− search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

71+ search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

7272 ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,

7373 preferences=Preferences(['oscar'], ['general'], engines, []))

7474 search = searx.search.Search(search_query)

@@ -77,7 +77,7 @@ class SearchTestCase(SearxTestCase):

7777

7878 def test_timeout_query_above_max(self):

7979 searx.search.max_request_timeout = 10.0

80− search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

80+ search_query = SearchQuery('test', [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

8181 ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0,

8282 preferences=Preferences(['oscar'], ['general'], engines, []))

8383 search = searx.search.Search(search_query)

@@ -85,7 +85,7 @@ class SearchTestCase(SearxTestCase):

8585 self.assertEqual(search.actual_timeout, 10.0)

8686

8787 def test_query_private_engine_without_token(self):

88− search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

88+ search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

8989 ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

9090 preferences=Preferences(['oscar'], ['general'], engines, []))

9191 search = searx.search.Search(search_query)

@@ -95,7 +95,7 @@ class SearchTestCase(SearxTestCase):

9595 def test_query_private_engine_with_incorrect_token(self):

9696 preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

9797 preferences_with_tokens.parse_dict({'tokens': 'bad-token'})

98− search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

98+ search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

9999 ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

100100 preferences=preferences_with_tokens)

101101 search = searx.search.Search(search_query)

@@ -105,7 +105,7 @@ class SearchTestCase(SearxTestCase):

105105 def test_query_private_engine_with_correct_token(self):

106106 preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

107107 preferences_with_tokens.parse_dict({'tokens': 'my-token'})

108− search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

108+ search_query = SearchQuery('test', [EngineRef(PRIVATE_ENGINE_NAME, 'general')],

109109 ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,

110110 preferences=preferences_with_tokens)

111111 search = searx.search.Search(search_query)

@@ -114,7 +114,7 @@ class SearchTestCase(SearxTestCase):

114114

115115 def test_external_bang(self):

116116 search_query = SearchQuery('yes yes',

117− [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

117+ [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

118118 ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

119119 preferences=Preferences(['oscar'], ['general'], engines, [],),

120120 external_bang="yt")

@@ -124,7 +124,7 @@ class SearchTestCase(SearxTestCase):

124124 self.assertTrue(results.redirect_url is not None)

125125

126126 search_query = SearchQuery('youtube never gonna give you up',

127− [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],

127+ [EngineRef(PUBLIC_ENGINE_NAME, 'general')],

128128 ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,

129129 preferences=Preferences(['oscar'], ['general'], engines, []),)

130130