@@ -15,27 +15,22 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
1515(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
1616'''
1717
18+import typing
1819import gc
19−import sys
2020import threading
2121from time import time
2222from uuid import uuid4
2323from _thread import start_new_thread
2424
25−from flask_babel import gettext
2625import requests.exceptions
2726import searx.poolrequests as requests_lib
28−from searx.engines import (
29− categories, engines, settings
30−)
27+from searx.engines import engines, settings
3128from searx.answerers import ask
3229from searx.external_bang import get_bang_url
3330from searx.utils import gen_useragent
34−from searx.query import RawTextQuery, SearchQuery, VALID_LANGUAGE_CODE
3531from searx.results import ResultContainer
3632from searx import logger
3733from searx.plugins import plugins
38−from searx.exceptions import SearxParameterException
3934
4035
4136logger = logger.getChild('search')
@@ -53,6 +48,49 @@ else:
5348 exit(1)
5449
5550
51+class EngineRef:
52+
53+ __slots__ = 'name', 'category', 'from_bang'
54+
55+ def __init__(self, name: str, category: str, from_bang: bool=False):
56+ self.name = name
57+ self.category = category
58+ self.from_bang = from_bang
59+
60+ def __str__(self):
61+ return "(" + self.name + "," + self.category + "," + str(self.from_bang) + ")"
62+
63+
64+class SearchQuery:
65+ """container for all the search parameters (query, language, etc...)"""
66+
67+ __slots__ = 'query', 'engineref_list', 'categories', 'lang', 'safesearch', 'pageno', 'time_range',\
68+ 'timeout_limit', 'external_bang'
69+
70+ def __init__(self,
71+ query: str,
72+ engineref_list: typing.List[EngineRef],
73+ categories: typing.List[str],
74+ lang: str,
75+ safesearch: bool,
76+ pageno: int,
77+ time_range: typing.Optional[str],
78+ timeout_limit: typing.Optional[float]=None,
79+ external_bang: typing.Optional[str]=False):
80+ self.query = query
81+ self.engineref_list = engineref_list
82+ self.categories = categories
83+ self.lang = lang
84+ self.safesearch = safesearch
85+ self.pageno = pageno
86+ self.time_range = time_range
87+ self.timeout_limit = timeout_limit
88+ self.external_bang = external_bang
89+
90+ def __str__(self):
91+ return self.query + ";" + str(self.engineref_list)
92+
93+
5694def send_http_request(engine, request_params):
5795 # create dictionary which contain all
5896 # informations about the request
@@ -247,167 +285,11 @@ def default_request_params():
247285 }
248286
249287
250−# remove duplicate queries.
251−# FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music'
252−def deduplicate_query_engines(query_engines):
253− uniq_query_engines = {q["category"] + '|' + q["name"]: q for q in query_engines}
254− return uniq_query_engines.values()
255−
256−
257−def get_search_query_from_webapp(preferences, form):
258− # no text for the query ?
259− if not form.get('q'):
260− raise SearxParameterException('q', '')
261−
262− # set blocked engines
263− disabled_engines = preferences.engines.get_disabled()
264−
265− # parse query, if tags are set, which change
266− # the serch engine or search-language
267− raw_text_query = RawTextQuery(form['q'], disabled_engines)
268−
269− # set query
270− query = raw_text_query.getQuery()
271−
272− # get and check page number
273− pageno_param = form.get('pageno', '1')
274− if not pageno_param.isdigit() or int(pageno_param) < 1:
275− raise SearxParameterException('pageno', pageno_param)
276− query_pageno = int(pageno_param)
277−
278− # get language
279− # set specific language if set on request, query or preferences
280− # TODO support search with multible languages
281− if len(raw_text_query.languages):
282− query_lang = raw_text_query.languages[-1]
283− elif 'language' in form:
284− query_lang = form.get('language')
285− else:
286− query_lang = preferences.get_value('language')
287−
288− # check language
289− if not VALID_LANGUAGE_CODE.match(query_lang):
290− raise SearxParameterException('language', query_lang)
291−
292− # get safesearch
293− if 'safesearch' in form:
294− query_safesearch = form.get('safesearch')
295− # first check safesearch
296− if not query_safesearch.isdigit():
297− raise SearxParameterException('safesearch', query_safesearch)
298− query_safesearch = int(query_safesearch)
299− else:
300− query_safesearch = preferences.get_value('safesearch')
301−
302− # safesearch : second check
303− if query_safesearch < 0 or query_safesearch > 2:
304− raise SearxParameterException('safesearch', query_safesearch)
305−
306− # get time_range
307− query_time_range = form.get('time_range')
308−
309− # check time_range
310− if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
311− raise SearxParameterException('time_range', query_time_range)
312−
313− # query_engines
314− query_engines = raw_text_query.engines
315−
316− # timeout_limit
317− query_timeout = raw_text_query.timeout_limit
318− if query_timeout is None and 'timeout_limit' in form:
319− raw_time_limit = form.get('timeout_limit')
320− if raw_time_limit in ['None', '']:
321− raw_time_limit = None
322− else:
323− try:
324− query_timeout = float(raw_time_limit)
325− except ValueError:
326− raise SearxParameterException('timeout_limit', raw_time_limit)
327−
328− # query_categories
329− query_categories = []
330−
331− # if engines are calculated from query,
332− # set categories by using that informations
333− if query_engines and raw_text_query.specific:
334− additional_categories = set()
335− for engine in query_engines:
336− if 'from_bang' in engine and engine['from_bang']:
337− additional_categories.add('none')
338− else:
339− additional_categories.add(engine['category'])
340− query_categories = list(additional_categories)
341−
342− # otherwise, using defined categories to
343− # calculate which engines should be used
344− else:
345− # set categories/engines
346− load_default_categories = True
347− for pd_name, pd in form.items():
348− if pd_name == 'categories':
349− query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories)
350− elif pd_name == 'engines':
351− pd_engines = [{'category': engines[engine].categories[0],
352− 'name': engine}
353− for engine in map(str.strip, pd.split(',')) if engine in engines]
354− if pd_engines:
355− query_engines.extend(pd_engines)
356− load_default_categories = False
357− elif pd_name.startswith('category_'):
358− category = pd_name[9:]
359−
360− # if category is not found in list, skip
361− if category not in categories:
362− continue
363−
364− if pd != 'off':
365− # add category to list
366− query_categories.append(category)
367− elif category in query_categories:
368− # remove category from list if property is set to 'off'
369− query_categories.remove(category)
370−
371− if not load_default_categories:
372− if not query_categories:
373− query_categories = list(set(engine['category']
374− for engine in query_engines))
375− else:
376− # if no category is specified for this search,
377− # using user-defined default-configuration which
378− # (is stored in cookie)
379− if not query_categories:
380− cookie_categories = preferences.get_value('categories')
381− for ccateg in cookie_categories:
382− if ccateg in categories:
383− query_categories.append(ccateg)
384−
385− # if still no category is specified, using general
386− # as default-category
387− if not query_categories:
388− query_categories = ['general']
389−
390− # using all engines for that search, which are
391− # declared under the specific categories
392− for categ in query_categories:
393− query_engines.extend({'category': categ,
394− 'name': engine.name}
395− for engine in categories[categ]
396− if (engine.name, categ) not in disabled_engines)
397−
398− query_engines = deduplicate_query_engines(query_engines)
399− external_bang = raw_text_query.external_bang
400−
401− return (SearchQuery(query, query_engines, query_categories,
402− query_lang, query_safesearch, query_pageno,
403− query_time_range, query_timeout, preferences,
404− external_bang=external_bang),
405− raw_text_query)
406−
407−
408288class Search:
409289 """Search information container"""
410290
291+ __slots__ = "search_query", "result_container", "start_time", "actual_timeout"
292+
411293 def __init__(self, search_query):
412294 # init vars
413295 super().__init__()
@@ -444,9 +326,6 @@ class Search:
444326 return False
445327
446328 def _is_accepted(self, engine_name, engine):
447− if not self.search_query.preferences.validate_token(engine):
448− return False
449−
450329 # skip suspended engines
451330 if engine.suspend_end_time >= time():
452331 logger.debug('Engine currently suspended: %s', engine_name)
@@ -462,13 +341,13 @@ class Search:
462341
463342 return True
464343
465− def _get_params(self, selected_engine, user_agent):
466− if selected_engine['name'] not in engines:
344+ def _get_params(self, engineref, user_agent):
345+ if engineref.name not in engines:
467346 return None, None
468347
469− engine = engines[selected_engine['name']]
348+ engine = engines[engineref.name]
470349
471− if not self._is_accepted(selected_engine['name'], engine):
350+ if not self._is_accepted(engineref.name, engine):
472351 return None, None
473352
474353 # set default request parameters
@@ -485,15 +364,13 @@ class Search:
485364 request_params['safesearch'] = self.search_query.safesearch
486365 request_params['time_range'] = self.search_query.time_range
487366
488− request_params['category'] = selected_engine['category']
367+ request_params['category'] = engineref.category
489368 request_params['pageno'] = self.search_query.pageno
490369
491370 return request_params, engine.timeout
492371
493372 # do search-request
494373 def _get_requests(self):
495− global number_of_searches
496−
497374 # init vars
498375 requests = []
499376
@@ -505,14 +382,14 @@ class Search:
505382 default_timeout = 0
506383
507384 # start search-reqest for all selected engines
508− for selected_engine in self.search_query.engines:
385+ for engineref in self.search_query.engineref_list:
509386 # set default request parameters
510− request_params, engine_timeout = self._get_params(selected_engine, user_agent)
387+ request_params, engine_timeout = self._get_params(engineref, user_agent)
511388 if request_params is None:
512389 continue
513390
514391 # append request to list
515− requests.append((selected_engine['name'], self.search_query.query, request_params))
392+ requests.append((engineref.name, self.search_query.query, request_params))
516393
517394 # update default_timeout
518395 default_timeout = max(default_timeout, engine_timeout)
@@ -535,7 +412,7 @@ class Search:
535412 actual_timeout = min(query_timeout, max_request_timeout)
536413
537414 logger.debug("actual_timeout={0} (default_timeout={1}, ?timeout_limit={2}, max_request_timeout={3})"
538− .format(self.actual_timeout, default_timeout, query_timeout, max_request_timeout))
415+ .format(actual_timeout, default_timeout, query_timeout, max_request_timeout))
539416
540417 return requests, actual_timeout
541418
@@ -567,6 +444,8 @@ class Search:
567444class SearchWithPlugins(Search):
568445 """Similar to the Search class but call the plugins."""
569446
447+ __slots__ = 'ordered_plugin_list', 'request'
448+
570449 def __init__(self, search_query, ordered_plugin_list, request):
571450 super().__init__(search_query)
572451 self.ordered_plugin_list = ordered_plugin_list