git.iko.soy/searx
on master help sign in
[enh] autocomplete refactoring, autocomplete on external bangs
63f17d2Alexandre Flament
searx/autocomplete.py+15−81
compare with master
unifiedsplit
@@ -20,97 +20,20 @@ from lxml import etree
20from json import loads
20from json import loads
21from urllib.parse import urlencode
21from urllib.parse import urlencode
22
22
23+from requests import RequestException
24+
23from searx import settings
25from searx import settings
24−from searx.languages import language_codes
25−from searx.engines import (
26− categories, engines, engine_shortcuts
27−)
28from searx.poolrequests import get as http_get
26from searx.poolrequests import get as http_get
27+from searx.exceptions import SearxEngineResponseException
29
28
30
29
31def get(*args, **kwargs):
30def get(*args, **kwargs):
32 if 'timeout' not in kwargs:
31 if 'timeout' not in kwargs:
33 kwargs['timeout'] = settings['outgoing']['request_timeout']
32 kwargs['timeout'] = settings['outgoing']['request_timeout']
34−
33+ kwargs['raise_for_httperror'] = True
35 return http_get(*args, **kwargs)
34 return http_get(*args, **kwargs)
36
35
37
36
38−def searx_bang(full_query):
39− '''check if the searchQuery contain a bang, and create fitting autocompleter results'''
40− # check if there is a query which can be parsed
41− if len(full_query.getQuery()) == 0:
42− return []
43−
44− results = []
45−
46− # check if current query stats with !bang
47− first_char = full_query.getQuery()[0]
48− if first_char == '!' or first_char == '?':
49− if len(full_query.getQuery()) == 1:
50− # show some example queries
51− # TODO, check if engine is not avaliable
52− results.append(first_char + "images")
53− results.append(first_char + "wikipedia")
54− results.append(first_char + "osm")
55− else:
56− engine_query = full_query.getQuery()[1:]
57−
58− # check if query starts with categorie name
59− for categorie in categories:
60− if categorie.startswith(engine_query):
61− results.append(first_char + '{categorie}'.format(categorie=categorie))
62−
63− # check if query starts with engine name
64− for engine in engines:
65− if engine.startswith(engine_query.replace('_', ' ')):
66− results.append(first_char + '{engine}'.format(engine=engine.replace(' ', '_')))
67−
68− # check if query starts with engine shortcut
69− for engine_shortcut in engine_shortcuts:
70− if engine_shortcut.startswith(engine_query):
71− results.append(first_char + '{engine_shortcut}'.format(engine_shortcut=engine_shortcut))
72−
73− # check if current query stats with :bang
74− elif first_char == ':':
75− if len(full_query.getQuery()) == 1:
76− # show some example queries
77− results.append(":en")
78− results.append(":en_us")
79− results.append(":english")
80− results.append(":united_kingdom")
81− else:
82− engine_query = full_query.getQuery()[1:]
83−
84− for lc in language_codes:
85− lang_id, lang_name, country, english_name = map(str.lower, lc)
86−
87− # check if query starts with language-id
88− if lang_id.startswith(engine_query):
89− if len(engine_query) <= 2:
90− results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
91− else:
92− results.append(':{lang_id}'.format(lang_id=lang_id))
93−
94− # check if query starts with language name
95− if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
96− results.append(':{lang_name}'.format(lang_name=lang_name))
97−
98− # check if query starts with country
99− if country.startswith(engine_query.replace('_', ' ')):
100− results.append(':{country}'.format(country=country.replace(' ', '_')))
101−
102− # remove duplicates
103− result_set = set(results)
104−
105− # remove results which are already contained in the query
106− for query_part in full_query.query_parts:
107− if query_part in result_set:
108− result_set.remove(query_part)
109−
110− # convert result_set back to list
111− return list(result_set)
112−
113−
114def dbpedia(query, lang):
37def dbpedia(query, lang):
115 # dbpedia autocompleter, no HTTPS
38 # dbpedia autocompleter, no HTTPS
116 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'
39 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'
@@ -204,3 +127,14 @@ backends = {'dbpedia': dbpedia,
204 'qwant': qwant,
127 'qwant': qwant,
205 'wikipedia': wikipedia
128 'wikipedia': wikipedia
206 }
129 }
130+
131+
132+def search_autocomplete(backend_name, query, lang):
133+ backend = backends.get(backend_name)
134+ if backend is None:
135+ return []
136+
137+ try:
138+ return backend(query, lang)
139+ except (RequestException, SearxEngineResponseException):
140+ return []
searx/query.py+286−118
@@ -1,162 +1,330 @@
1−#!/usr/bin/env python
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−
3−'''
4−searx is free software: you can redistribute it and/or modify
5−it under the terms of the GNU Affero General Public License as published by
6−the Free Software Foundation, either version 3 of the License, or
7−(at your option) any later version.
8−
9−searx is distributed in the hope that it will be useful,
10−but WITHOUT ANY WARRANTY; without even the implied warranty of
11−MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12−GNU Affero General Public License for more details.
13−
14−You should have received a copy of the GNU Affero General Public License
15−along with searx. If not, see < http://www.gnu.org/licenses/ >.
16−
17−(C) 2014 by Thomas Pointhuber, <thomas.pointhuber@gmx.at>
18−'''
19
2
3+from abc import abstractmethod, ABC
20import re
4import re
21
5
22from searx.languages import language_codes
6from searx.languages import language_codes
23from searx.engines import categories, engines, engine_shortcuts
7from searx.engines import categories, engines, engine_shortcuts
8+from searx.external_bang import get_bang_definition_and_autocomplete
24from searx.search import EngineRef
9from searx.search import EngineRef
25from searx.webutils import VALID_LANGUAGE_CODE
10from searx.webutils import VALID_LANGUAGE_CODE
26
11
27
12
13+class QueryPartParser(ABC):
14+
15+ __slots__ = "raw_text_query", "enable_autocomplete"
16+
17+ @staticmethod
18+ @abstractmethod
19+ def check(raw_value):
20+ """Check if raw_value can be parsed"""
21+
22+ def __init__(self, raw_text_query, enable_autocomplete):
23+ self.raw_text_query = raw_text_query
24+ self.enable_autocomplete = enable_autocomplete
25+
26+ @abstractmethod
27+ def __call__(self, raw_value):
28+ """Try to parse raw_value: set the self.raw_text_query properties
29+
30+ return True if raw_value has been parsed
31+
32+ self.raw_text_query.autocomplete_list is also modified
33+ if self.enable_autocomplete is True
34+ """
35+
36+ def _add_autocomplete(self, value):
37+ if value not in self.raw_text_query.autocomplete_list:
38+ self.raw_text_query.autocomplete_list.append(value)
39+
40+
41+class TimeoutParser(QueryPartParser):
42+
43+ @staticmethod
44+ def check(raw_value):
45+ return raw_value[0] == '<'
46+
47+ def __call__(self, raw_value):
48+ value = raw_value[1:]
49+ found = self._parse(value) if len(value) > 0 else False
50+ if self.enable_autocomplete and not value:
51+ self._autocomplete()
52+ return found
53+
54+ def _parse(self, value):
55+ if not value.isdigit():
56+ return False
57+ raw_timeout_limit = int(value)
58+ if raw_timeout_limit < 100:
59+ # below 100, the unit is the second ( <3 = 3 seconds timeout )
60+ self.raw_text_query.timeout_limit = float(raw_timeout_limit)
61+ else:
62+ # 100 or above, the unit is the millisecond ( <850 = 850 milliseconds timeout )
63+ self.raw_text_query.timeout_limit = raw_timeout_limit / 1000.0
64+ return True
65+
66+ def _autocomplete(self):
67+ for suggestion in ['<3', '<850']:
68+ self._add_autocomplete(suggestion)
69+
70+
71+class LanguageParser(QueryPartParser):
72+
73+ @staticmethod
74+ def check(raw_value):
75+ return raw_value[0] == ':'
76+
77+ def __call__(self, raw_value):
78+ value = raw_value[1:].lower().replace('_', '-')
79+ found = self._parse(value) if len(value) > 0 else False
80+ if self.enable_autocomplete and not found:
81+ self._autocomplete(value)
82+ return found
83+
84+ def _parse(self, value):
85+ found = False
86+ # check if any language-code is equal with
87+ # declared language-codes
88+ for lc in language_codes:
89+ lang_id, lang_name, country, english_name = map(str.lower, lc)
90+
91+ # if correct language-code is found
92+ # set it as new search-language
93+
94+ if (value == lang_id
95+ or value == lang_name
96+ or value == english_name
97+ or value.replace('-', ' ') == country)\
98+ and value not in self.raw_text_query.languages:
99+ found = True
100+ lang_parts = lang_id.split('-')
101+ if len(lang_parts) == 2:
102+ self.raw_text_query.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())
103+ else:
104+ self.raw_text_query.languages.append(lang_id)
105+ # to ensure best match (first match is not necessarily the best one)
106+ if value == lang_id:
107+ break
108+
109+ # user may set a valid, yet not selectable language
110+ if VALID_LANGUAGE_CODE.match(value):
111+ lang_parts = value.split('-')
112+ if len(lang_parts) > 1:
113+ value = lang_parts[0].lower() + '-' + lang_parts[1].upper()
114+ if value not in self.raw_text_query.languages:
115+ self.raw_text_query.languages.append(value)
116+ found = True
117+
118+ return found
119+
120+ def _autocomplete(self, value):
121+ if not value:
122+ # show some example queries
123+ for lang in [":en", ":en_us", ":english", ":united_kingdom"]:
124+ self.raw_text_query.autocomplete_list.append(lang)
125+ return
126+
127+ for lc in language_codes:
128+ lang_id, lang_name, country, english_name = map(str.lower, lc)
129+
130+ # check if query starts with language-id
131+ if lang_id.startswith(value):
132+ if len(value) <= 2:
133+ self._add_autocomplete(':' + lang_id.split('-')[0])
134+ else:
135+ self._add_autocomplete(':' + lang_id)
136+
137+ # check if query starts with language name
138+ if lang_name.startswith(value) or english_name.startswith(value):
139+ self._add_autocomplete(':' + lang_name)
140+
141+ # check if query starts with country
142+ # here "new_zealand" is "new-zealand" (see __call__)
143+ if country.startswith(value.replace('-', ' ')):
144+ self._add_autocomplete(':' + country.replace(' ', '_'))
145+
146+
147+class ExternalBangParser(QueryPartParser):
148+
149+ @staticmethod
150+ def check(raw_value):
151+ return raw_value.startswith('!!')
152+
153+ def __call__(self, raw_value):
154+ value = raw_value[2:]
155+ found, bang_ac_list = self._parse(value) if len(value) > 0 else (False, [])
156+ if self.enable_autocomplete:
157+ self._autocomplete(bang_ac_list)
158+ return found
159+
160+ def _parse(self, value):
161+ found = False
162+ bang_definition, bang_ac_list = get_bang_definition_and_autocomplete(value)
163+ if bang_definition is not None:
164+ self.raw_text_query.external_bang = value
165+ found = True
166+ return found, bang_ac_list
167+
168+ def _autocomplete(self, bang_ac_list):
169+ if not bang_ac_list:
170+ bang_ac_list = ['g', 'ddg', 'bing']
171+ for external_bang in bang_ac_list:
172+ self._add_autocomplete('!!' + external_bang)
173+
174+
175+class BangParser(QueryPartParser):
176+
177+ @staticmethod
178+ def check(raw_value):
179+ return raw_value[0] == '!' or raw_value[0] == '?'
180+
181+ def __call__(self, raw_value):
182+ value = raw_value[1:].replace('-', ' ').replace('_', ' ')
183+ found = self._parse(value) if len(value) > 0 else False
184+ if found and raw_value[0] == '!':
185+ self.raw_text_query.specific = True
186+ if self.enable_autocomplete:
187+ self._autocomplete(raw_value[0], value)
188+ return found
189+
190+ def _parse(self, value):
191+ # check if prefix is equal with engine shortcut
192+ if value in engine_shortcuts:
193+ value = engine_shortcuts[value]
194+
195+ # check if prefix is equal with engine name
196+ if value in engines:
197+ self.raw_text_query.enginerefs.append(EngineRef(value, 'none'))
198+ return True
199+
200+ # check if prefix is equal with categorie name
201+ if value in categories:
202+ # using all engines for that search, which
203+ # are declared under that categorie name
204+ self.raw_text_query.enginerefs.extend(EngineRef(engine.name, value)
205+ for engine in categories[value]
206+ if (engine.name, value) not in self.raw_text_query.disabled_engines)
207+ return True
208+
209+ return False
210+
211+ def _autocomplete(self, first_char, value):
212+ if not value:
213+ # show some example queries
214+ for suggestion in ['images', 'wikipedia', 'osm']:
215+ if suggestion not in self.raw_text_query.disabled_engines or suggestion in categories:
216+ self._add_autocomplete(first_char + suggestion)
217+ return
218+
219+ # check if query starts with categorie name
220+ for category in categories:
221+ if category.startswith(value):
222+ self._add_autocomplete(first_char + category)
223+
224+ # check if query starts with engine name
225+ for engine in engines:
226+ if engine.startswith(value):
227+ self._add_autocomplete(first_char + engine.replace(' ', '_'))
228+
229+ # check if query starts with engine shortcut
230+ for engine_shortcut in engine_shortcuts:
231+ if engine_shortcut.startswith(value):
232+ self._add_autocomplete(first_char + engine_shortcut)
233+
234+
28class RawTextQuery:
235class RawTextQuery:
29 """parse raw text query (the value from the html input)"""
236 """parse raw text query (the value from the html input)"""
30
237
238+ PARSER_CLASSES = [
239+ TimeoutParser, # this force the timeout
240+ LanguageParser, # this force a language
241+ ExternalBangParser, # external bang (must be before BangParser)
242+ BangParser # this force a engine or category
243+ ]
244+
31 def __init__(self, query, disabled_engines):
245 def __init__(self, query, disabled_engines):
32 assert isinstance(query, str)
246 assert isinstance(query, str)
247+ # input parameters
33 self.query = query
248 self.query = query
34− self.disabled_engines = []
249+ self.disabled_engines = disabled_engines if disabled_engines else []
35−
250+ # parsed values
36− if disabled_engines:
37− self.disabled_engines = disabled_engines
38−
39− self.query_parts = []
40− self.user_query_parts = []
41 self.enginerefs = []
251 self.enginerefs = []
42 self.languages = []
252 self.languages = []
43 self.timeout_limit = None
253 self.timeout_limit = None
44 self.external_bang = None
254 self.external_bang = None
45 self.specific = False
255 self.specific = False
256+ self.autocomplete_list = []
257+ # internal properties
258+ self.query_parts = [] # use self.getFullQuery()
259+ self.user_query_parts = [] # use self.getQuery()
260+ self.autocomplete_location = None
46 self._parse_query()
261 self._parse_query()
47
262
48− # parse query, if tags are set, which
49− # change the search engine or search-language
50 def _parse_query(self):
263 def _parse_query(self):
51− self.query_parts = []
264+ """
265+ parse self.query, if tags are set, which
266+ change the search engine or search-language
267+ """
52
268
53 # split query, including whitespaces
269 # split query, including whitespaces
54 raw_query_parts = re.split(r'(\s+)', self.query)
270 raw_query_parts = re.split(r'(\s+)', self.query)
55
271
56− for query_part in raw_query_parts:
272+ last_index_location = None
57− searx_query_part = False
273+ autocomplete_index = len(raw_query_parts) - 1
58
274
275+ for i, query_part in enumerate(raw_query_parts):
59 # part does only contain spaces, skip
276 # part does only contain spaces, skip
60 if query_part.isspace()\
277 if query_part.isspace()\
61 or query_part == '':
278 or query_part == '':
62 continue
279 continue
63
280
64− # this force the timeout
281+ # parse special commands
65− if query_part[0] == '<':
282+ special_part = False
66− try:
283+ for parser_class in RawTextQuery.PARSER_CLASSES:
67− raw_timeout_limit = int(query_part[1:])
68− if raw_timeout_limit < 100:
69− # below 100, the unit is the second ( <3 = 3 seconds timeout )
70− self.timeout_limit = float(raw_timeout_limit)
71− else:
72− # 100 or above, the unit is the millisecond ( <850 = 850 milliseconds timeout )
73− self.timeout_limit = raw_timeout_limit / 1000.0
74− searx_query_part = True
75− except ValueError:
76− # error not reported to the user
77− pass
78−
79− # this force a language
80− if query_part[0] == ':' and len(query_part) > 1:
284+ if parser_class.check(query_part):
81− lang = query_part[1:].lower().replace('_', '-')
285+ special_part = parser_class(self, i == autocomplete_index)(query_part)
82−
83− # check if any language-code is equal with
84− # declared language-codes
85− for lc in language_codes:
86− lang_id, lang_name, country, english_name = map(str.lower, lc)
87−
88− # if correct language-code is found
89− # set it as new search-language
90− if (lang == lang_id
91− or lang == lang_name
92− or lang == english_name
93− or lang.replace('-', ' ') == country)\
94− and lang not in self.languages:
95− searx_query_part = True
96− lang_parts = lang_id.split('-')
97− if len(lang_parts) == 2:
98− self.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())
99− else:
100− self.languages.append(lang_id)
101− # to ensure best match (first match is not necessarily the best one)
102− if lang == lang_id:
103− break
286+ break
104−
105− # user may set a valid, yet not selectable language
106− if VALID_LANGUAGE_CODE.match(lang):
107− lang_parts = lang.split('-')
108− if len(lang_parts) > 1:
109− lang = lang_parts[0].lower() + '-' + lang_parts[1].upper()
110− if lang not in self.languages:
111− self.languages.append(lang)
112− searx_query_part = True
113−
114− # external bang
115− if query_part[0:2] == "!!":
116− self.external_bang = query_part[2:]
117− searx_query_part = True
118− continue
119− # this force a engine or category
120− if query_part[0] == '!' or query_part[0] == '?':
121− prefix = query_part[1:].replace('-', ' ').replace('_', ' ')
122−
123− # check if prefix is equal with engine shortcut
124− if prefix in engine_shortcuts:
125− searx_query_part = True
126− engine_name = engine_shortcuts[prefix]
127− if engine_name in engines:
128− self.enginerefs.append(EngineRef(engine_name, 'none'))
129−
130− # check if prefix is equal with engine name
131− elif prefix in engines:
132− searx_query_part = True
133− self.enginerefs.append(EngineRef(prefix, 'none'))
134−
135− # check if prefix is equal with categorie name
136− elif prefix in categories:
137− # using all engines for that search, which
138− # are declared under that categorie name
139− searx_query_part = True
140− self.enginerefs.extend(EngineRef(engine.name, prefix)
141− for engine in categories[prefix]
142− if (engine.name, prefix) not in self.disabled_engines)
143−
144− if query_part[0] == '!':
145− self.specific = True
146
287
147 # append query part to query_part list
288 # append query part to query_part list
148− if searx_query_part:
289+ qlist = self.query_parts if special_part else self.user_query_parts
149− self.query_parts.append(query_part)
150− else:
151− self.user_query_parts.append(query_part)
290+ qlist.append(query_part)
291+ last_index_location = (qlist, len(qlist) - 1)
292+
293+ self.autocomplete_location = last_index_location
294+
295+ def get_autocomplete_full_query(self, text):
296+ qlist, position = self.autocomplete_location
297+ qlist[position] = text
298+ return self.getFullQuery()
152
299
153 def changeQuery(self, query):
300 def changeQuery(self, query):
154 self.user_query_parts = query.strip().split()
301 self.user_query_parts = query.strip().split()
302+ self.query = self.getFullQuery()
303+ self.autocomplete_location = (self.user_query_parts, len(self.user_query_parts) - 1)
304+ self.autocomplete_list = []
155 return self
305 return self
156
306
157 def getQuery(self):
307 def getQuery(self):
158 return ' '.join(self.user_query_parts)
308 return ' '.join(self.user_query_parts)
159
309
160 def getFullQuery(self):
310 def getFullQuery(self):
311+ """
161− # get full querry including whitespaces
312+ get full querry including whitespaces
313+ """
162− return '{0} {1}'.format(''.join(self.query_parts), self.getQuery()).strip()
314+ return '{0} {1}'.format(' '.join(self.query_parts), self.getQuery()).strip()
315+
316+ def __str__(self):
317+ return self.getFullQuery()
318+
319+ def __repr__(self):
320+ return f"<{self.__class__.__name__} " \
321+ + f"query={self.query!r} " \
322+ + f"disabled_engines={self.disabled_engines!r}\n " \
323+ + f"languages={self.languages!r} " \
324+ + f"timeout_limit={self.timeout_limit!r} "\
325+ + f"external_bang={self.external_bang!r} " \
326+ + f"specific={self.specific!r} " \
327+ + f"enginerefs={self.enginerefs!r}\n " \
328+ + f"autocomplete_list={self.autocomplete_list!r}\n " \
329+ + f"query_parts={self.query_parts!r}\n " \
330+ + f"user_query_parts={self.user_query_parts!r} >"
searx/webapp.py+19−24
@@ -74,12 +74,13 @@ from searx.languages import language_codes as languages
74from searx.search import SearchWithPlugins, initialize as search_initialize
74from searx.search import SearchWithPlugins, initialize as search_initialize
75from searx.search.checker import get_result as checker_get_result
75from searx.search.checker import get_result as checker_get_result
76from searx.query import RawTextQuery
76from searx.query import RawTextQuery
77−from searx.autocomplete import searx_bang, backends as autocomplete_backends
77+from searx.autocomplete import search_autocomplete, backends as autocomplete_backends
78from searx.plugins import plugins
78from searx.plugins import plugins
79from searx.plugins.oa_doi_rewrite import get_doi_resolver
79from searx.plugins.oa_doi_rewrite import get_doi_resolver
80from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
80from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
81from searx.answerers import answerers
81from searx.answerers import answerers
82from searx.poolrequests import get_global_proxies
82from searx.poolrequests import get_global_proxies
83+from searx.answerers import ask
83from searx.metrology.error_recorder import errors_per_engines
84from searx.metrology.error_recorder import errors_per_engines
84
85
85# serve pages with HTTP/1.1
86# serve pages with HTTP/1.1
@@ -763,27 +764,18 @@ def about():
763def autocompleter():
764def autocompleter():
764 """Return autocompleter results"""
765 """Return autocompleter results"""
765
766
767+ # run autocompleter
768+ results = []
769+
766 # set blocked engines
770 # set blocked engines
767 disabled_engines = request.preferences.engines.get_disabled()
771 disabled_engines = request.preferences.engines.get_disabled()
768
772
769 # parse query
773 # parse query
770 raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
774 raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
771
775
772− # check if search query is set
773− if not raw_text_query.getQuery():
774− return '', 400
775−
776− # run autocompleter
777− completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
778−
779− # parse searx specific autocompleter results like !bang
780− raw_results = searx_bang(raw_text_query)
781−
782 # normal autocompletion results only appear if no inner results returned
776 # normal autocompletion results only appear if no inner results returned
783− # and there is a query part besides the engine and language bangs
777+ # and there is a query part
784− if len(raw_results) == 0 and completer and (len(raw_text_query.query_parts) > 1 or
778+ if len(raw_text_query.autocomplete_list) == 0 and len(raw_text_query.getQuery()) > 0:
785− (len(raw_text_query.languages) == 0 and
786− not raw_text_query.specific)):
787 # get language from cookie
779 # get language from cookie
788 language = request.preferences.get_value('language')
780 language = request.preferences.get_value('language')
789 if not language or language == 'all':
781 if not language or language == 'all':
@@ -791,15 +783,18 @@ def autocompleter():
791 else:
783 else:
792 language = language.split('-')[0]
784 language = language.split('-')[0]
793 # run autocompletion
785 # run autocompletion
786+ raw_results = search_autocomplete(request.preferences.get_value('autocomplete'),
794− raw_results.extend(completer(raw_text_query.getQuery(), language))
787+ raw_text_query.getQuery(), language)
795−
788+ for result in raw_results:
796− # parse results (write :language and !engine back to result string)
789+ results.append(raw_text_query.changeQuery(result).getFullQuery())
797− results = []
790+
798− for result in raw_results:
791+ if len(raw_text_query.autocomplete_list) > 0:
799− raw_text_query.changeQuery(result)
792+ for autocomplete_text in raw_text_query.autocomplete_list:
800−
793+ results.append(raw_text_query.get_autocomplete_full_query(autocomplete_text))
801− # add parsed result
794+
802− results.append(raw_text_query.getFullQuery())
795+ for answers in ask(raw_text_query):
796+ for answer in answers:
797+ results.append(str(answer['answer']))
803
798
804 # return autocompleter results
799 # return autocompleter results
805 if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
800 if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
tests/unit/test_query.py+178
@@ -1,7 +1,20 @@
1+from searx.search import initialize
1from searx.query import RawTextQuery
2from searx.query import RawTextQuery
2from searx.testing import SearxTestCase
3from searx.testing import SearxTestCase
3
4
4
5
6+TEST_ENGINES = [
7+ {
8+ 'name': 'dummy engine',
9+ 'engine': 'dummy',
10+ 'categories': 'general',
11+ 'shortcut': 'du',
12+ 'timeout': 3.0,
13+ 'tokens': [],
14+ },
15+]
16+
17+
5class TestQuery(SearxTestCase):
18class TestQuery(SearxTestCase):
6
19
7 def test_simple_query(self):
20 def test_simple_query(self):
@@ -14,6 +27,37 @@ class TestQuery(SearxTestCase):
14 self.assertEqual(len(query.languages), 0)
27 self.assertEqual(len(query.languages), 0)
15 self.assertFalse(query.specific)
28 self.assertFalse(query.specific)
16
29
30+ def test_multiple_spaces_query(self):
31+ query_text = '\tthe query'
32+ query = RawTextQuery(query_text, [])
33+
34+ self.assertEqual(query.getFullQuery(), 'the query')
35+ self.assertEqual(len(query.query_parts), 0)
36+ self.assertEqual(len(query.user_query_parts), 2)
37+ self.assertEqual(len(query.languages), 0)
38+ self.assertFalse(query.specific)
39+
40+ def test_str_method(self):
41+ query_text = '<7 the query'
42+ query = RawTextQuery(query_text, [])
43+ self.assertEqual(str(query), '<7 the query')
44+
45+ def test_repr_method(self):
46+ query_text = '<8 the query'
47+ query = RawTextQuery(query_text, [])
48+ r = repr(query)
49+ self.assertTrue(r.startswith(f"<RawTextQuery query='{query_text}' "))
50+
51+ def test_change_query(self):
52+ query_text = '<8 the query'
53+ query = RawTextQuery(query_text, [])
54+ another_query = query.changeQuery('another text')
55+ self.assertEqual(query, another_query)
56+ self.assertEqual(query.getFullQuery(), '<8 another text')
57+
58+
59+class TestLanguageParser(SearxTestCase):
60+
17 def test_language_code(self):
61 def test_language_code(self):
18 language = 'es-ES'
62 language = 'es-ES'
19 query_text = 'the query'
63 query_text = 'the query'
@@ -68,6 +112,30 @@ class TestQuery(SearxTestCase):
68 self.assertEqual(len(query.languages), 0)
112 self.assertEqual(len(query.languages), 0)
69 self.assertFalse(query.specific)
113 self.assertFalse(query.specific)
70
114
115+ def test_autocomplete_empty(self):
116+ query_text = 'the query :'
117+ query = RawTextQuery(query_text, [])
118+ self.assertEqual(query.autocomplete_list, [":en", ":en_us", ":english", ":united_kingdom"])
119+
120+ def test_autocomplete(self):
121+ query = RawTextQuery(':englis', [])
122+ self.assertEqual(query.autocomplete_list, [":english"])
123+
124+ query = RawTextQuery(':deutschla', [])
125+ self.assertEqual(query.autocomplete_list, [":deutschland"])
126+
127+ query = RawTextQuery(':new_zea', [])
128+ self.assertEqual(query.autocomplete_list, [":new_zealand"])
129+
130+ query = RawTextQuery(':hu-H', [])
131+ self.assertEqual(query.autocomplete_list, [":hu-hu"])
132+
133+ query = RawTextQuery(':v', [])
134+ self.assertEqual(query.autocomplete_list, [":vi", ":tiếng việt"])
135+
136+
137+class TestTimeoutParser(SearxTestCase):
138+
71 def test_timeout_below100(self):
139 def test_timeout_below100(self):
72 query_text = '<3 the query'
140 query_text = '<3 the query'
73 query = RawTextQuery(query_text, [])
141 query = RawTextQuery(query_text, [])
@@ -105,3 +173,113 @@ class TestQuery(SearxTestCase):
105 self.assertEqual(query.getQuery(), query_text)
173 self.assertEqual(query.getQuery(), query_text)
106 self.assertEqual(query.timeout_limit, None)
174 self.assertEqual(query.timeout_limit, None)
107 self.assertFalse(query.specific)
175 self.assertFalse(query.specific)
176+
177+ def test_timeout_autocomplete(self):
178+ # invalid number: it is not bang but it is part of the query
179+ query_text = 'the query <'
180+ query = RawTextQuery(query_text, [])
181+
182+ self.assertEqual(query.getFullQuery(), query_text)
183+ self.assertEqual(len(query.query_parts), 0)
184+ self.assertEqual(query.getQuery(), query_text)
185+ self.assertEqual(query.timeout_limit, None)
186+ self.assertFalse(query.specific)
187+ self.assertEqual(query.autocomplete_list, ['<3', '<850'])
188+
189+
190+class TestExternalBangParser(SearxTestCase):
191+
192+ def test_external_bang(self):
193+ query_text = '!!ddg the query'
194+ query = RawTextQuery(query_text, [])
195+
196+ self.assertEqual(query.getFullQuery(), query_text)
197+ self.assertEqual(len(query.query_parts), 1)
198+ self.assertFalse(query.specific)
199+
200+ def test_external_bang_not_found(self):
201+ query_text = '!!notfoundbang the query'
202+ query = RawTextQuery(query_text, [])
203+
204+ self.assertEqual(query.getFullQuery(), query_text)
205+ self.assertEqual(query.external_bang, None)
206+ self.assertFalse(query.specific)
207+
208+ def test_external_bang_autocomplete(self):
209+ query_text = 'the query !!dd'
210+ query = RawTextQuery(query_text, [])
211+
212+ self.assertEqual(query.getFullQuery(), '!!dd the query')
213+ self.assertEqual(len(query.query_parts), 1)
214+ self.assertFalse(query.specific)
215+ self.assertGreater(len(query.autocomplete_list), 0)
216+
217+ a = query.autocomplete_list[0]
218+ self.assertEqual(query.get_autocomplete_full_query(a), a + ' the query')
219+
220+ def test_external_bang_autocomplete_empty(self):
221+ query_text = 'the query !!'
222+ query = RawTextQuery(query_text, [])
223+
224+ self.assertEqual(query.getFullQuery(), 'the query !!')
225+ self.assertEqual(len(query.query_parts), 0)
226+ self.assertFalse(query.specific)
227+ self.assertGreater(len(query.autocomplete_list), 2)
228+
229+ a = query.autocomplete_list[0]
230+ self.assertEqual(query.get_autocomplete_full_query(a), 'the query ' + a)
231+
232+
233+class TestBang(SearxTestCase):
234+
235+ SPECIFIC_BANGS = ['!dummy_engine', '!du', '!general']
236+ NOT_SPECIFIC_BANGS = ['?dummy_engine', '?du', '?general']
237+ THE_QUERY = 'the query'
238+
239+ def test_bang(self):
240+ initialize(TEST_ENGINES)
241+
242+ for bang in TestBang.SPECIFIC_BANGS + TestBang.NOT_SPECIFIC_BANGS:
243+ with self.subTest(msg="Check bang", bang=bang):
244+ query_text = TestBang.THE_QUERY + ' ' + bang
245+ query = RawTextQuery(query_text, [])
246+
247+ self.assertEqual(query.getFullQuery(), bang + ' ' + TestBang.THE_QUERY)
248+ self.assertEqual(query.query_parts, [bang])
249+ self.assertEqual(query.user_query_parts, TestBang.THE_QUERY.split(' '))
250+
251+ def test_specific(self):
252+ for bang in TestBang.SPECIFIC_BANGS:
253+ with self.subTest(msg="Check bang is specific", bang=bang):
254+ query_text = TestBang.THE_QUERY + ' ' + bang
255+ query = RawTextQuery(query_text, [])
256+ self.assertTrue(query.specific)
257+
258+ def test_not_specific(self):
259+ for bang in TestBang.NOT_SPECIFIC_BANGS:
260+ with self.subTest(msg="Check bang is not specific", bang=bang):
261+ query_text = TestBang.THE_QUERY + ' ' + bang
262+ query = RawTextQuery(query_text, [])
263+ self.assertFalse(query.specific)
264+
265+ def test_bang_not_found(self):
266+ initialize(TEST_ENGINES)
267+ query = RawTextQuery('the query !bang_not_found', [])
268+ self.assertEqual(query.getFullQuery(), 'the query !bang_not_found')
269+
270+ def test_bang_autocomplete(self):
271+ initialize(TEST_ENGINES)
272+ query = RawTextQuery('the query !dum', [])
273+ self.assertEqual(query.autocomplete_list, ['!dummy_engine'])
274+
275+ query = RawTextQuery('!dum the query', [])
276+ self.assertEqual(query.autocomplete_list, [])
277+ self.assertEqual(query.getQuery(), '!dum the query')
278+
279+ def test_bang_autocomplete_empty(self):
280+ initialize()
281+ query = RawTextQuery('the query !', [])
282+ self.assertEqual(query.autocomplete_list, ['!images', '!wikipedia', '!osm'])
283+
284+ query = RawTextQuery('the query ?', ['osm'])
285+ self.assertEqual(query.autocomplete_list, ['?images', '?wikipedia'])