git.iko.soy/searx
on master help sign in
[mod] add searx/webadapter.py

* move searx.search.get_search_query_from_webapp to searx.webadapter * move searx.query.SearchQuery to searx.search

commit2dbc0de0cd000459ebfdb3d015be8684e737e95c authorAlexandre Flament <alex@al-f.net> dateTue 22 Sep 2020, 13:59 (6 years ago) parent21dbc7e
6 files changed+221−220
compare with master
unifiedsplit
searx/query.py−20 searx/search.py+21−165 searx/webadapter.py+162 searx/webapp.py+2−1 tests/unit/test_search.py+34−33 utils/standalone_searx.py+2−1
searx/query.py−20
@@ -178,23 +178,3 @@ class RawTextQuery:
178178 def getFullQuery(self):
179179 # get full querry including whitespaces
180180 return ''.join(self.query_parts)
181−
182−
183−class SearchQuery:
184− """container for all the search parameters (query, language, etc...)"""
185−
186− def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range,
187− timeout_limit=None, preferences=None, external_bang=None):
188− self.query = query
189− self.engines = engines
190− self.categories = categories
191− self.lang = lang
192− self.safesearch = safesearch
193− self.pageno = pageno
194− self.time_range = None if time_range in ('', 'None', None) else time_range
195− self.timeout_limit = timeout_limit
196− self.preferences = preferences
197− self.external_bang = external_bang
198−
199− def __str__(self):
200− return self.query + ";" + str(self.engines)
searx/search.py+21−165
@@ -16,26 +16,20 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
1616'''
1717
1818import gc
19−import sys
2019import threading
2120from time import time
2221from uuid import uuid4
2322from _thread import start_new_thread
2423
25−from flask_babel import gettext
2624import requests.exceptions
2725import searx.poolrequests as requests_lib
28−from searx.engines import (
29− categories, engines, settings
30−)
26+from searx.engines import engines, settings
3127from searx.answerers import ask
3228from searx.external_bang import get_bang_url
3329from searx.utils import gen_useragent
34−from searx.query import RawTextQuery, SearchQuery, VALID_LANGUAGE_CODE
3530from searx.results import ResultContainer
3631from searx import logger
3732from searx.plugins import plugins
38−from searx.exceptions import SearxParameterException
3933
4034
4135logger = logger.getChild('search')
@@ -53,6 +47,26 @@ else:
5347 exit(1)
5448
5549
50+class SearchQuery:
51+ """container for all the search parameters (query, language, etc...)"""
52+
53+ def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range,
54+ timeout_limit=None, preferences=None, external_bang=None):
55+ self.query = query
56+ self.engines = engines
57+ self.categories = categories
58+ self.lang = lang
59+ self.safesearch = safesearch
60+ self.pageno = pageno
61+ self.time_range = None if time_range in ('', 'None', None) else time_range
62+ self.timeout_limit = timeout_limit
63+ self.preferences = preferences
64+ self.external_bang = external_bang
65+
66+ def __str__(self):
67+ return self.query + ";" + str(self.engines)
68+
69+
5670def send_http_request(engine, request_params):
5771 # create dictionary which contain all
5872 # informations about the request
@@ -247,164 +261,6 @@ def default_request_params():
247261 }
248262
249263
250−# remove duplicate queries.
251−# FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music'
252−def deduplicate_query_engines(query_engines):
253− uniq_query_engines = {q["category"] + '|' + q["name"]: q for q in query_engines}
254− return uniq_query_engines.values()
255−
256−
257−def get_search_query_from_webapp(preferences, form):
258− # no text for the query ?
259− if not form.get('q'):
260− raise SearxParameterException('q', '')
261−
262− # set blocked engines
263− disabled_engines = preferences.engines.get_disabled()
264−
265− # parse query, if tags are set, which change
266− # the serch engine or search-language
267− raw_text_query = RawTextQuery(form['q'], disabled_engines)
268−
269− # set query
270− query = raw_text_query.getQuery()
271−
272− # get and check page number
273− pageno_param = form.get('pageno', '1')
274− if not pageno_param.isdigit() or int(pageno_param) < 1:
275− raise SearxParameterException('pageno', pageno_param)
276− query_pageno = int(pageno_param)
277−
278− # get language
279− # set specific language if set on request, query or preferences
280− # TODO support search with multible languages
281− if len(raw_text_query.languages):
282− query_lang = raw_text_query.languages[-1]
283− elif 'language' in form:
284− query_lang = form.get('language')
285− else:
286− query_lang = preferences.get_value('language')
287−
288− # check language
289− if not VALID_LANGUAGE_CODE.match(query_lang):
290− raise SearxParameterException('language', query_lang)
291−
292− # get safesearch
293− if 'safesearch' in form:
294− query_safesearch = form.get('safesearch')
295− # first check safesearch
296− if not query_safesearch.isdigit():
297− raise SearxParameterException('safesearch', query_safesearch)
298− query_safesearch = int(query_safesearch)
299− else:
300− query_safesearch = preferences.get_value('safesearch')
301−
302− # safesearch : second check
303− if query_safesearch < 0 or query_safesearch > 2:
304− raise SearxParameterException('safesearch', query_safesearch)
305−
306− # get time_range
307− query_time_range = form.get('time_range')
308−
309− # check time_range
310− if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
311− raise SearxParameterException('time_range', query_time_range)
312−
313− # query_engines
314− query_engines = raw_text_query.engines
315−
316− # timeout_limit
317− query_timeout = raw_text_query.timeout_limit
318− if query_timeout is None and 'timeout_limit' in form:
319− raw_time_limit = form.get('timeout_limit')
320− if raw_time_limit in ['None', '']:
321− raw_time_limit = None
322− else:
323− try:
324− query_timeout = float(raw_time_limit)
325− except ValueError:
326− raise SearxParameterException('timeout_limit', raw_time_limit)
327−
328− # query_categories
329− query_categories = []
330−
331− # if engines are calculated from query,
332− # set categories by using that informations
333− if query_engines and raw_text_query.specific:
334− additional_categories = set()
335− for engine in query_engines:
336− if 'from_bang' in engine and engine['from_bang']:
337− additional_categories.add('none')
338− else:
339− additional_categories.add(engine['category'])
340− query_categories = list(additional_categories)
341−
342− # otherwise, using defined categories to
343− # calculate which engines should be used
344− else:
345− # set categories/engines
346− load_default_categories = True
347− for pd_name, pd in form.items():
348− if pd_name == 'categories':
349− query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories)
350− elif pd_name == 'engines':
351− pd_engines = [{'category': engines[engine].categories[0],
352− 'name': engine}
353− for engine in map(str.strip, pd.split(',')) if engine in engines]
354− if pd_engines:
355− query_engines.extend(pd_engines)
356− load_default_categories = False
357− elif pd_name.startswith('category_'):
358− category = pd_name[9:]
359−
360− # if category is not found in list, skip
361− if category not in categories:
362− continue
363−
364− if pd != 'off':
365− # add category to list
366− query_categories.append(category)
367− elif category in query_categories:
368− # remove category from list if property is set to 'off'
369− query_categories.remove(category)
370−
371− if not load_default_categories:
372− if not query_categories:
373− query_categories = list(set(engine['category']
374− for engine in query_engines))
375− else:
376− # if no category is specified for this search,
377− # using user-defined default-configuration which
378− # (is stored in cookie)
379− if not query_categories:
380− cookie_categories = preferences.get_value('categories')
381− for ccateg in cookie_categories:
382− if ccateg in categories:
383− query_categories.append(ccateg)
384−
385− # if still no category is specified, using general
386− # as default-category
387− if not query_categories:
388− query_categories = ['general']
389−
390− # using all engines for that search, which are
391− # declared under the specific categories
392− for categ in query_categories:
393− query_engines.extend({'category': categ,
394− 'name': engine.name}
395− for engine in categories[categ]
396− if (engine.name, categ) not in disabled_engines)
397−
398− query_engines = deduplicate_query_engines(query_engines)
399− external_bang = raw_text_query.external_bang
400−
401− return (SearchQuery(query, query_engines, query_categories,
402− query_lang, query_safesearch, query_pageno,
403− query_time_range, query_timeout, preferences,
404− external_bang=external_bang),
405− raw_text_query)
406−
407−
408264class Search:
409265 """Search information container"""
410266
searx/webadapter.pynew file+162
@@ -0,0 +1,162 @@
1+from searx.exceptions import SearxParameterException
2+from searx.query import RawTextQuery, VALID_LANGUAGE_CODE
3+from searx.engines import categories, engines
4+from searx.search import SearchQuery
5+
6+
7+# remove duplicate queries.
8+# FIXME: does not fix "!music !soundcloud", because the categories are 'none' and 'music'
9+def deduplicate_query_engines(query_engines):
10+ uniq_query_engines = {q["category"] + '|' + q["name"]: q for q in query_engines}
11+ return uniq_query_engines.values()
12+
13+
14+def get_search_query_from_webapp(preferences, form):
15+ # no text for the query ?
16+ if not form.get('q'):
17+ raise SearxParameterException('q', '')
18+
19+ # set blocked engines
20+ disabled_engines = preferences.engines.get_disabled()
21+
22+ # parse query, if tags are set, which change
23+ # the serch engine or search-language
24+ raw_text_query = RawTextQuery(form['q'], disabled_engines)
25+
26+ # set query
27+ query = raw_text_query.getQuery()
28+
29+ # get and check page number
30+ pageno_param = form.get('pageno', '1')
31+ if not pageno_param.isdigit() or int(pageno_param) < 1:
32+ raise SearxParameterException('pageno', pageno_param)
33+ query_pageno = int(pageno_param)
34+
35+ # get language
36+ # set specific language if set on request, query or preferences
37+ # TODO support search with multible languages
38+ if len(raw_text_query.languages):
39+ query_lang = raw_text_query.languages[-1]
40+ elif 'language' in form:
41+ query_lang = form.get('language')
42+ else:
43+ query_lang = preferences.get_value('language')
44+
45+ # check language
46+ if not VALID_LANGUAGE_CODE.match(query_lang):
47+ raise SearxParameterException('language', query_lang)
48+
49+ # get safesearch
50+ if 'safesearch' in form:
51+ query_safesearch = form.get('safesearch')
52+ # first check safesearch
53+ if not query_safesearch.isdigit():
54+ raise SearxParameterException('safesearch', query_safesearch)
55+ query_safesearch = int(query_safesearch)
56+ else:
57+ query_safesearch = preferences.get_value('safesearch')
58+
59+ # safesearch : second check
60+ if query_safesearch < 0 or query_safesearch > 2:
61+ raise SearxParameterException('safesearch', query_safesearch)
62+
63+ # get time_range
64+ query_time_range = form.get('time_range')
65+
66+ # check time_range
67+ if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
68+ raise SearxParameterException('time_range', query_time_range)
69+
70+ # query_engines
71+ query_engines = raw_text_query.engines
72+
73+ # timeout_limit
74+ query_timeout = raw_text_query.timeout_limit
75+ if query_timeout is None and 'timeout_limit' in form:
76+ raw_time_limit = form.get('timeout_limit')
77+ if raw_time_limit in ['None', '']:
78+ raw_time_limit = None
79+ else:
80+ try:
81+ query_timeout = float(raw_time_limit)
82+ except ValueError:
83+ raise SearxParameterException('timeout_limit', raw_time_limit)
84+
85+ # query_categories
86+ query_categories = []
87+
88+ # if engines are calculated from query,
89+ # set categories by using that informations
90+ if query_engines and raw_text_query.specific:
91+ additional_categories = set()
92+ for engine in query_engines:
93+ if 'from_bang' in engine and engine['from_bang']:
94+ additional_categories.add('none')
95+ else:
96+ additional_categories.add(engine['category'])
97+ query_categories = list(additional_categories)
98+
99+ # otherwise, using defined categories to
100+ # calculate which engines should be used
101+ else:
102+ # set categories/engines
103+ load_default_categories = True
104+ for pd_name, pd in form.items():
105+ if pd_name == 'categories':
106+ query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories)
107+ elif pd_name == 'engines':
108+ pd_engines = [{'category': engines[engine].categories[0],
109+ 'name': engine}
110+ for engine in map(str.strip, pd.split(',')) if engine in engines]
111+ if pd_engines:
112+ query_engines.extend(pd_engines)
113+ load_default_categories = False
114+ elif pd_name.startswith('category_'):
115+ category = pd_name[9:]
116+
117+ # if category is not found in list, skip
118+ if category not in categories:
119+ continue
120+
121+ if pd != 'off':
122+ # add category to list
123+ query_categories.append(category)
124+ elif category in query_categories:
125+ # remove category from list if property is set to 'off'
126+ query_categories.remove(category)
127+
128+ if not load_default_categories:
129+ if not query_categories:
130+ query_categories = list(set(engine['category']
131+ for engine in query_engines))
132+ else:
133+ # if no category is specified for this search,
134+ # using user-defined default-configuration which
135+ # (is stored in cookie)
136+ if not query_categories:
137+ cookie_categories = preferences.get_value('categories')
138+ for ccateg in cookie_categories:
139+ if ccateg in categories:
140+ query_categories.append(ccateg)
141+
142+ # if still no category is specified, using general
143+ # as default-category
144+ if not query_categories:
145+ query_categories = ['general']
146+
147+ # using all engines for that search, which are
148+ # declared under the specific categories
149+ for categ in query_categories:
150+ query_engines.extend({'category': categ,
151+ 'name': engine.name}
152+ for engine in categories[categ]
153+ if (engine.name, categ) not in disabled_engines)
154+
155+ query_engines = deduplicate_query_engines(query_engines)
156+ external_bang = raw_text_query.external_bang
157+
158+ return (SearchQuery(query, query_engines, query_categories,
159+ query_lang, query_safesearch, query_pageno,
160+ query_time_range, query_timeout, preferences,
161+ external_bang=external_bang),
162+ raw_text_query)
searx/webapp.py+2−1
@@ -67,10 +67,11 @@ from searx.webutils import (
6767 get_static_files, get_result_templates, get_themes,
6868 prettify_url, new_hmac
6969)
70+from searx.webadapter import get_search_query_from_webapp
7071from searx.utils import html_to_text, gen_useragent, dict_subset, match_language
7172from searx.version import VERSION_STRING
7273from searx.languages import language_codes as languages
73−from searx.search import SearchWithPlugins, get_search_query_from_webapp
74+from searx.search import SearchWithPlugins
7475from searx.query import RawTextQuery
7576from searx.autocomplete import searx_bang, backends as autocomplete_backends
7677from searx.plugins import plugins
tests/unit/test_search.py+34−33
@@ -5,6 +5,7 @@ from searx.preferences import Preferences
55from searx.engines import engines
66
77import searx.search
8+from searx.search import SearchQuery
89
910
1011SAFESEARCH = 0
@@ -40,53 +41,53 @@ class SearchTestCase(SearxTestCase):
4041
4142 def test_timeout_simple(self):
4243 searx.search.max_request_timeout = None
43− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
44− ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,
45− preferences=Preferences(['oscar'], ['general'], engines, []))
44+ search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
45+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,
46+ preferences=Preferences(['oscar'], ['general'], engines, []))
4647 search = searx.search.Search(search_query)
4748 search.search()
4849 self.assertEqual(search.actual_timeout, 3.0)
4950
5051 def test_timeout_query_above_default_nomax(self):
5152 searx.search.max_request_timeout = None
52− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
53− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,
54− preferences=Preferences(['oscar'], ['general'], engines, []))
53+ search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
54+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,
55+ preferences=Preferences(['oscar'], ['general'], engines, []))
5556 search = searx.search.Search(search_query)
5657 search.search()
5758 self.assertEqual(search.actual_timeout, 3.0)
5859
5960 def test_timeout_query_below_default_nomax(self):
6061 searx.search.max_request_timeout = None
61− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
62− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0,
63− preferences=Preferences(['oscar'], ['general'], engines, []))
62+ search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
63+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 1.0,
64+ preferences=Preferences(['oscar'], ['general'], engines, []))
6465 search = searx.search.Search(search_query)
6566 search.search()
6667 self.assertEqual(search.actual_timeout, 1.0)
6768
6869 def test_timeout_query_below_max(self):
6970 searx.search.max_request_timeout = 10.0
70− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
71− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,
72− preferences=Preferences(['oscar'], ['general'], engines, []))
71+ search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
72+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 5.0,
73+ preferences=Preferences(['oscar'], ['general'], engines, []))
7374 search = searx.search.Search(search_query)
7475 search.search()
7576 self.assertEqual(search.actual_timeout, 5.0)
7677
7778 def test_timeout_query_above_max(self):
7879 searx.search.max_request_timeout = 10.0
79− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
80− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0,
81− preferences=Preferences(['oscar'], ['general'], engines, []))
80+ search_query = SearchQuery('test', [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
81+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 15.0,
82+ preferences=Preferences(['oscar'], ['general'], engines, []))
8283 search = searx.search.Search(search_query)
8384 search.search()
8485 self.assertEqual(search.actual_timeout, 10.0)
8586
8687 def test_query_private_engine_without_token(self):
87− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],
88− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,
89− preferences=Preferences(['oscar'], ['general'], engines, []))
88+ search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],
89+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,
90+ preferences=Preferences(['oscar'], ['general'], engines, []))
9091 search = searx.search.Search(search_query)
9192 results = search.search()
9293 self.assertEqual(results.results_length(), 0)
@@ -94,9 +95,9 @@ class SearchTestCase(SearxTestCase):
9495 def test_query_private_engine_with_incorrect_token(self):
9596 preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])
9697 preferences_with_tokens.parse_dict({'tokens': 'bad-token'})
97− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],
98− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,
99− preferences=preferences_with_tokens)
98+ search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],
99+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,
100+ preferences=preferences_with_tokens)
100101 search = searx.search.Search(search_query)
101102 results = search.search()
102103 self.assertEqual(results.results_length(), 0)
@@ -104,28 +105,28 @@ class SearchTestCase(SearxTestCase):
104105 def test_query_private_engine_with_correct_token(self):
105106 preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])
106107 preferences_with_tokens.parse_dict({'tokens': 'my-token'})
107− search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],
108− ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,
109− preferences=preferences_with_tokens)
108+ search_query = SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],
109+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, 2.0,
110+ preferences=preferences_with_tokens)
110111 search = searx.search.Search(search_query)
111112 results = search.search()
112113 self.assertEqual(results.results_length(), 1)
113114
114115 def test_external_bang(self):
115− search_query = searx.query.SearchQuery('yes yes',
116− [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
117− ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,
118− preferences=Preferences(['oscar'], ['general'], engines, [],),
119− external_bang="yt")
116+ search_query = SearchQuery('yes yes',
117+ [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
118+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,
119+ preferences=Preferences(['oscar'], ['general'], engines, [],),
120+ external_bang="yt")
120121 search = searx.search.Search(search_query)
121122 results = search.search()
122123 # For checking if the user redirected with the youtube external bang
123124 self.assertTrue(results.redirect_url is not None)
124125
125− search_query = searx.query.SearchQuery('youtube never gonna give you up',
126− [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
127− ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,
128− preferences=Preferences(['oscar'], ['general'], engines, []),)
126+ search_query = SearchQuery('youtube never gonna give you up',
127+ [{'category': 'general', 'name': PUBLIC_ENGINE_NAME}],
128+ ['general'], 'en-US', SAFESEARCH, PAGENO, None, None,
129+ preferences=Preferences(['oscar'], ['general'], engines, []),)
129130
130131 search = searx.search.Search(search_query)
131132 results = search.search()
utils/standalone_searx.py+2−1
@@ -31,6 +31,7 @@ import searx.query
3131import searx.search
3232import searx.engines
3333import searx.preferences
34+import searx.webadapter
3435import argparse
3536
3637searx.engines.initialize_engines(settings['engines'])
@@ -64,7 +65,7 @@ form = {
6465preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, [])
6566preferences.key_value_settings['safesearch'].parse(args.safesearch)
6667
67−search_query, raw_text_query = searx.search.get_search_query_from_webapp(preferences, form)
68+search_query, raw_text_query = searx.webadapter.get_search_query_from_webapp(preferences, form)
6869search = searx.search.Search(search_query)
6970result_container = search.search()
7071