on master help sign in
[mod] searx.search.Search: split the search method into different methods
commitc4067a9d02d97ca08259daeede92a031cd608724 authorAlexandre Flament <alex@al-f.net> dateMon 14 Sep 2020, 13:21 (6 years ago) parentad0758e
1 file changed+96−57
searx/search.py+96−57
searx/search.py+96−57
@@ -414,30 +414,86 @@ class Search:
414414 super().__init__()
415415 self.search_query = search_query
416416 self.result_container = ResultContainer()
417+ self.start_time = None
417418 self.actual_timeout = None
418419
419− # do search-request
420− def search(self):
421− global number_of_searches
422−
423− # Check if there is a external bang. After that we can stop because the search will terminate.
420+ def search_external_bang(self):
421+ """
422+ Check if there is a external bang.
423+ If yes, update self.result_container and return True
424+ """
424425 if self.search_query.external_bang:
425426 self.result_container.redirect_url = get_bang_url(self.search_query)
426427
427428 # This means there was a valid bang and the
428429 # rest of the search does not need to be continued
429430 if isinstance(self.result_container.redirect_url, str):
430− return self.result_container
431− # start time
432− start_time = time()
433−
434− # answeres ?
431+ return True
432+ return False
433+
434+ def search_answerers(self):
435+ """
436+ Check if an answer return a result.
437+ If yes, update self.result_container and return True
438+ """
435439 answerers_results = ask(self.search_query)
436440
437441 if answerers_results:
438442 for results in answerers_results:
439443 self.result_container.extend('answer', results)
440− return self.result_container
444+ return True
445+ return False
446+
447+ def _is_accepted(self, engine_name, engine):
448+ if not self.search_query.preferences.validate_token(engine):
449+ return False
450+
451+ # skip suspended engines
452+ if engine.suspend_end_time >= time():
453+ logger.debug('Engine currently suspended: %s', engine_name)
454+ return False
455+
456+ # if paging is not supported, skip
457+ if self.search_query.pageno > 1 and not engine.paging:
458+ return False
459+
460+ # if time_range is not supported, skip
461+ if self.search_query.time_range and not engine.time_range_support:
462+ return False
463+
464+ return True
465+
466+ def _get_params(self, selected_engine, user_agent):
467+ if selected_engine['name'] not in engines:
468+ return None, None
469+
470+ engine = engines[selected_engine['name']]
471+
472+ if not self._is_accepted(selected_engine['name'], engine):
473+ return None, None
474+
475+ # set default request parameters
476+ request_params = {}
477+ if not engine.offline:
478+ request_params = default_request_params()
479+ request_params['headers']['User-Agent'] = user_agent
480+
481+ if hasattr(engine, 'language') and engine.language:
482+ request_params['language'] = engine.language
483+ else:
484+ request_params['language'] = self.search_query.lang
485+
486+ request_params['safesearch'] = self.search_query.safesearch
487+ request_params['time_range'] = self.search_query.time_range
488+
489+ request_params['category'] = selected_engine['category']
490+ request_params['pageno'] = self.search_query.pageno
491+
492+ return request_params, engine.timeout
493+
494+ # do search-request
495+ def _get_requests(self):
496+ global number_of_searches
441497
442498 # init vars
443499 requests = []
@@ -449,59 +505,24 @@ class Search:
449505 # user_agent = request.headers.get('User-Agent', '')
450506 user_agent = gen_useragent()
451507
452− search_query = self.search_query
453−
454508 # max of all selected engine timeout
455509 default_timeout = 0
456510
457511 # start search-reqest for all selected engines
458− for selected_engine in search_query.engines:
459− if selected_engine['name'] not in engines:
460− continue
461−
462− engine = engines[selected_engine['name']]
463−
464− if not search_query.preferences.validate_token(engine):
465− continue
466−
467− # skip suspended engines
468− if engine.suspend_end_time >= time():
469− logger.debug('Engine currently suspended: %s', selected_engine['name'])
470− continue
471−
472− # if paging is not supported, skip
473− if search_query.pageno > 1 and not engine.paging:
474− continue
475−
476− # if time_range is not supported, skip
477− if search_query.time_range and not engine.time_range_support:
478− continue
479−
512+ for selected_engine in self.search_query.engines:
480513 # set default request parameters
481− request_params = {}
482− if not engine.offline:
483− request_params = default_request_params()
484− request_params['headers']['User-Agent'] = user_agent
485−
486− if hasattr(engine, 'language') and engine.language:
487− request_params['language'] = engine.language
488− else:
489− request_params['language'] = search_query.lang
490−
491− request_params['safesearch'] = search_query.safesearch
492− request_params['time_range'] = search_query.time_range
493−
494− request_params['category'] = selected_engine['category']
495− request_params['pageno'] = search_query.pageno
514+ request_params, engine_timeout = self._get_params(selected_engine, user_agent)
515+ if request_params is None:
516+ continue
496517
497518 # append request to list
498− requests.append((selected_engine['name'], search_query.query, request_params))
519+ requests.append((selected_engine['name'], self.search_query.query, request_params))
499520
500521 # update default_timeout
501− default_timeout = max(default_timeout, engine.timeout)
522+ default_timeout = max(default_timeout, engine_timeout)
502523
503524 # adjust timeout
504− self.actual_timeout = default_timeout
525+ actual_timeout = default_timeout
505526 query_timeout = self.search_query.timeout_limit
506527
507528 if max_request_timeout is None and query_timeout is None:
@@ -509,23 +530,41 @@ class Search:
509530 pass
510531 elif max_request_timeout is None and query_timeout is not None:
511532 # No max, but user query: From user query except if above default
512− self.actual_timeout = min(default_timeout, query_timeout)
533+ actual_timeout = min(default_timeout, query_timeout)
513534 elif max_request_timeout is not None and query_timeout is None:
514535 # Max, no user query: Default except if above max
515− self.actual_timeout = min(default_timeout, max_request_timeout)
536+ actual_timeout = min(default_timeout, max_request_timeout)
516537 elif max_request_timeout is not None and query_timeout is not None:
517538 # Max & user query: From user query except if above max
518− self.actual_timeout = min(query_timeout, max_request_timeout)
539+ actual_timeout = min(query_timeout, max_request_timeout)
519540
520541 logger.debug("actual_timeout={0} (default_timeout={1}, ?timeout_limit={2}, max_request_timeout={3})"
521542 .format(self.actual_timeout, default_timeout, query_timeout, max_request_timeout))
522543
544+ return requests, actual_timeout
545+
546+ def search_standard(self):
547+ """
548+ Update self.result_container, self.actual_timeout
549+ """
550+ requests, self.actual_timeout = self._get_requests()
551+
523552 # send all search-request
524553 if requests:
525− search_multiple_requests(requests, self.result_container, start_time, self.actual_timeout)
554+ search_multiple_requests(requests, self.result_container, self.start_time, self.actual_timeout)
526555 start_new_thread(gc.collect, tuple())
527556
528557 # return results, suggestions, answers and infoboxes
558+ return True
559+
560+ # do search-request
561+ def search(self):
562+ self.start_time = time()
563+
564+ if not self.search_external_bang():
565+ if not self.search_answerers():
566+ self.search_standard()
567+
529568 return self.result_container
530569
531570