@@ -1,162 +1,330 @@
1−#!/usr/bin/env python
2−
3−'''
4−searx is free software: you can redistribute it and/or modify
5−it under the terms of the GNU Affero General Public License as published by
6−the Free Software Foundation, either version 3 of the License, or
7−(at your option) any later version.
8−
9−searx is distributed in the hope that it will be useful,
10−but WITHOUT ANY WARRANTY; without even the implied warranty of
11−MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12−GNU Affero General Public License for more details.
13−
14−You should have received a copy of the GNU Affero General Public License
15−along with searx. If not, see < http://www.gnu.org/licenses/ >.
16−
17−(C) 2014 by Thomas Pointhuber, <thomas.pointhuber@gmx.at>
18−'''
1+# SPDX-License-Identifier: AGPL-3.0-or-later
192
3+from abc import abstractmethod, ABC
204import re
215
226from searx.languages import language_codes
237from searx.engines import categories, engines, engine_shortcuts
8+from searx.external_bang import get_bang_definition_and_autocomplete
249from searx.search import EngineRef
2510from searx.webutils import VALID_LANGUAGE_CODE
2611
2712
13+class QueryPartParser(ABC):
14+
15+ __slots__ = "raw_text_query", "enable_autocomplete"
16+
17+ @staticmethod
18+ @abstractmethod
19+ def check(raw_value):
20+ """Check if raw_value can be parsed"""
21+
22+ def __init__(self, raw_text_query, enable_autocomplete):
23+ self.raw_text_query = raw_text_query
24+ self.enable_autocomplete = enable_autocomplete
25+
26+ @abstractmethod
27+ def __call__(self, raw_value):
28+ """Try to parse raw_value: set the self.raw_text_query properties
29+
30+ return True if raw_value has been parsed
31+
32+ self.raw_text_query.autocomplete_list is also modified
33+ if self.enable_autocomplete is True
34+ """
35+
36+ def _add_autocomplete(self, value):
37+ if value not in self.raw_text_query.autocomplete_list:
38+ self.raw_text_query.autocomplete_list.append(value)
39+
40+
41+class TimeoutParser(QueryPartParser):
42+
43+ @staticmethod
44+ def check(raw_value):
45+ return raw_value[0] == '<'
46+
47+ def __call__(self, raw_value):
48+ value = raw_value[1:]
49+ found = self._parse(value) if len(value) > 0 else False
50+ if self.enable_autocomplete and not value:
51+ self._autocomplete()
52+ return found
53+
54+ def _parse(self, value):
55+ if not value.isdigit():
56+ return False
57+ raw_timeout_limit = int(value)
58+ if raw_timeout_limit < 100:
59+ # below 100, the unit is the second ( <3 = 3 seconds timeout )
60+ self.raw_text_query.timeout_limit = float(raw_timeout_limit)
61+ else:
62+ # 100 or above, the unit is the millisecond ( <850 = 850 milliseconds timeout )
63+ self.raw_text_query.timeout_limit = raw_timeout_limit / 1000.0
64+ return True
65+
66+ def _autocomplete(self):
67+ for suggestion in ['<3', '<850']:
68+ self._add_autocomplete(suggestion)
69+
70+
71+class LanguageParser(QueryPartParser):
72+
73+ @staticmethod
74+ def check(raw_value):
75+ return raw_value[0] == ':'
76+
77+ def __call__(self, raw_value):
78+ value = raw_value[1:].lower().replace('_', '-')
79+ found = self._parse(value) if len(value) > 0 else False
80+ if self.enable_autocomplete and not found:
81+ self._autocomplete(value)
82+ return found
83+
84+ def _parse(self, value):
85+ found = False
86+ # check if any language-code is equal with
87+ # declared language-codes
88+ for lc in language_codes:
89+ lang_id, lang_name, country, english_name = map(str.lower, lc)
90+
91+ # if correct language-code is found
92+ # set it as new search-language
93+
94+ if (value == lang_id
95+ or value == lang_name
96+ or value == english_name
97+ or value.replace('-', ' ') == country)\
98+ and value not in self.raw_text_query.languages:
99+ found = True
100+ lang_parts = lang_id.split('-')
101+ if len(lang_parts) == 2:
102+ self.raw_text_query.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())
103+ else:
104+ self.raw_text_query.languages.append(lang_id)
105+ # to ensure best match (first match is not necessarily the best one)
106+ if value == lang_id:
107+ break
108+
109+ # user may set a valid, yet not selectable language
110+ if VALID_LANGUAGE_CODE.match(value):
111+ lang_parts = value.split('-')
112+ if len(lang_parts) > 1:
113+ value = lang_parts[0].lower() + '-' + lang_parts[1].upper()
114+ if value not in self.raw_text_query.languages:
115+ self.raw_text_query.languages.append(value)
116+ found = True
117+
118+ return found
119+
120+ def _autocomplete(self, value):
121+ if not value:
122+ # show some example queries
123+ for lang in [":en", ":en_us", ":english", ":united_kingdom"]:
124+ self.raw_text_query.autocomplete_list.append(lang)
125+ return
126+
127+ for lc in language_codes:
128+ lang_id, lang_name, country, english_name = map(str.lower, lc)
129+
130+ # check if query starts with language-id
131+ if lang_id.startswith(value):
132+ if len(value) <= 2:
133+ self._add_autocomplete(':' + lang_id.split('-')[0])
134+ else:
135+ self._add_autocomplete(':' + lang_id)
136+
137+ # check if query starts with language name
138+ if lang_name.startswith(value) or english_name.startswith(value):
139+ self._add_autocomplete(':' + lang_name)
140+
141+ # check if query starts with country
142+ # here "new_zealand" is "new-zealand" (see __call__)
143+ if country.startswith(value.replace('-', ' ')):
144+ self._add_autocomplete(':' + country.replace(' ', '_'))
145+
146+
147+class ExternalBangParser(QueryPartParser):
148+
149+ @staticmethod
150+ def check(raw_value):
151+ return raw_value.startswith('!!')
152+
153+ def __call__(self, raw_value):
154+ value = raw_value[2:]
155+ found, bang_ac_list = self._parse(value) if len(value) > 0 else (False, [])
156+ if self.enable_autocomplete:
157+ self._autocomplete(bang_ac_list)
158+ return found
159+
160+ def _parse(self, value):
161+ found = False
162+ bang_definition, bang_ac_list = get_bang_definition_and_autocomplete(value)
163+ if bang_definition is not None:
164+ self.raw_text_query.external_bang = value
165+ found = True
166+ return found, bang_ac_list
167+
168+ def _autocomplete(self, bang_ac_list):
169+ if not bang_ac_list:
170+ bang_ac_list = ['g', 'ddg', 'bing']
171+ for external_bang in bang_ac_list:
172+ self._add_autocomplete('!!' + external_bang)
173+
174+
175+class BangParser(QueryPartParser):
176+
177+ @staticmethod
178+ def check(raw_value):
179+ return raw_value[0] == '!' or raw_value[0] == '?'
180+
181+ def __call__(self, raw_value):
182+ value = raw_value[1:].replace('-', ' ').replace('_', ' ')
183+ found = self._parse(value) if len(value) > 0 else False
184+ if found and raw_value[0] == '!':
185+ self.raw_text_query.specific = True
186+ if self.enable_autocomplete:
187+ self._autocomplete(raw_value[0], value)
188+ return found
189+
190+ def _parse(self, value):
191+ # check if prefix is equal with engine shortcut
192+ if value in engine_shortcuts:
193+ value = engine_shortcuts[value]
194+
195+ # check if prefix is equal with engine name
196+ if value in engines:
197+ self.raw_text_query.enginerefs.append(EngineRef(value, 'none'))
198+ return True
199+
200+ # check if prefix is equal with categorie name
201+ if value in categories:
202+ # using all engines for that search, which
203+ # are declared under that categorie name
204+ self.raw_text_query.enginerefs.extend(EngineRef(engine.name, value)
205+ for engine in categories[value]
206+ if (engine.name, value) not in self.raw_text_query.disabled_engines)
207+ return True
208+
209+ return False
210+
211+ def _autocomplete(self, first_char, value):
212+ if not value:
213+ # show some example queries
214+ for suggestion in ['images', 'wikipedia', 'osm']:
215+ if suggestion not in self.raw_text_query.disabled_engines or suggestion in categories:
216+ self._add_autocomplete(first_char + suggestion)
217+ return
218+
219+ # check if query starts with categorie name
220+ for category in categories:
221+ if category.startswith(value):
222+ self._add_autocomplete(first_char + category)
223+
224+ # check if query starts with engine name
225+ for engine in engines:
226+ if engine.startswith(value):
227+ self._add_autocomplete(first_char + engine.replace(' ', '_'))
228+
229+ # check if query starts with engine shortcut
230+ for engine_shortcut in engine_shortcuts:
231+ if engine_shortcut.startswith(value):
232+ self._add_autocomplete(first_char + engine_shortcut)
233+
234+
28235class RawTextQuery:
29236 """parse raw text query (the value from the html input)"""
30237
238+ PARSER_CLASSES = [
239+ TimeoutParser, # this force the timeout
240+ LanguageParser, # this force a language
241+ ExternalBangParser, # external bang (must be before BangParser)
242+ BangParser # this force a engine or category
243+ ]
244+
31245 def __init__(self, query, disabled_engines):
32246 assert isinstance(query, str)
247+ # input parameters
33248 self.query = query
34− self.disabled_engines = []
35−
36− if disabled_engines:
37− self.disabled_engines = disabled_engines
38−
39− self.query_parts = []
40− self.user_query_parts = []
249+ self.disabled_engines = disabled_engines if disabled_engines else []
250+ # parsed values
41251 self.enginerefs = []
42252 self.languages = []
43253 self.timeout_limit = None
44254 self.external_bang = None
45255 self.specific = False
256+ self.autocomplete_list = []
257+ # internal properties
258+ self.query_parts = [] # use self.getFullQuery()
259+ self.user_query_parts = [] # use self.getQuery()
260+ self.autocomplete_location = None
46261 self._parse_query()
47262
48− # parse query, if tags are set, which
49− # change the search engine or search-language
50263 def _parse_query(self):
51− self.query_parts = []
264+ """
265+ parse self.query, if tags are set, which
266+ change the search engine or search-language
267+ """
52268
53269 # split query, including whitespaces
54270 raw_query_parts = re.split(r'(\s+)', self.query)
55271
56− for query_part in raw_query_parts:
57− searx_query_part = False
272+ last_index_location = None
273+ autocomplete_index = len(raw_query_parts) - 1
58274
275+ for i, query_part in enumerate(raw_query_parts):
59276 # part does only contain spaces, skip
60277 if query_part.isspace()\
61278 or query_part == '':
62279 continue
63280
64− # this force the timeout
65− if query_part[0] == '<':
66− try:
67− raw_timeout_limit = int(query_part[1:])
68− if raw_timeout_limit < 100:
69− # below 100, the unit is the second ( <3 = 3 seconds timeout )
70− self.timeout_limit = float(raw_timeout_limit)
71− else:
72− # 100 or above, the unit is the millisecond ( <850 = 850 milliseconds timeout )
73− self.timeout_limit = raw_timeout_limit / 1000.0
74− searx_query_part = True
75− except ValueError:
76− # error not reported to the user
77− pass
78−
79− # this force a language
80− if query_part[0] == ':' and len(query_part) > 1:
81− lang = query_part[1:].lower().replace('_', '-')
82−
83− # check if any language-code is equal with
84− # declared language-codes
85− for lc in language_codes:
86− lang_id, lang_name, country, english_name = map(str.lower, lc)
87−
88− # if correct language-code is found
89− # set it as new search-language
90− if (lang == lang_id
91− or lang == lang_name
92− or lang == english_name
93− or lang.replace('-', ' ') == country)\
94− and lang not in self.languages:
95− searx_query_part = True
96− lang_parts = lang_id.split('-')
97− if len(lang_parts) == 2:
98− self.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())
99− else:
100− self.languages.append(lang_id)
101− # to ensure best match (first match is not necessarily the best one)
102− if lang == lang_id:
103− break
104−
105− # user may set a valid, yet not selectable language
106− if VALID_LANGUAGE_CODE.match(lang):
107− lang_parts = lang.split('-')
108− if len(lang_parts) > 1:
109− lang = lang_parts[0].lower() + '-' + lang_parts[1].upper()
110− if lang not in self.languages:
111− self.languages.append(lang)
112− searx_query_part = True
113−
114− # external bang
115− if query_part[0:2] == "!!":
116− self.external_bang = query_part[2:]
117− searx_query_part = True
118− continue
119− # this force a engine or category
120− if query_part[0] == '!' or query_part[0] == '?':
121− prefix = query_part[1:].replace('-', ' ').replace('_', ' ')
122−
123− # check if prefix is equal with engine shortcut
124− if prefix in engine_shortcuts:
125− searx_query_part = True
126− engine_name = engine_shortcuts[prefix]
127− if engine_name in engines:
128− self.enginerefs.append(EngineRef(engine_name, 'none'))
129−
130− # check if prefix is equal with engine name
131− elif prefix in engines:
132− searx_query_part = True
133− self.enginerefs.append(EngineRef(prefix, 'none'))
134−
135− # check if prefix is equal with categorie name
136− elif prefix in categories:
137− # using all engines for that search, which
138− # are declared under that categorie name
139− searx_query_part = True
140− self.enginerefs.extend(EngineRef(engine.name, prefix)
141− for engine in categories[prefix]
142− if (engine.name, prefix) not in self.disabled_engines)
143−
144− if query_part[0] == '!':
145− self.specific = True
281+ # parse special commands
282+ special_part = False
283+ for parser_class in RawTextQuery.PARSER_CLASSES:
284+ if parser_class.check(query_part):
285+ special_part = parser_class(self, i == autocomplete_index)(query_part)
286+ break
146287
147288 # append query part to query_part list
148− if searx_query_part:
149− self.query_parts.append(query_part)
150− else:
151− self.user_query_parts.append(query_part)
289+ qlist = self.query_parts if special_part else self.user_query_parts
290+ qlist.append(query_part)
291+ last_index_location = (qlist, len(qlist) - 1)
292+
293+ self.autocomplete_location = last_index_location
294+
295+ def get_autocomplete_full_query(self, text):
296+ qlist, position = self.autocomplete_location
297+ qlist[position] = text
298+ return self.getFullQuery()
152299
153300 def changeQuery(self, query):
154301 self.user_query_parts = query.strip().split()
302+ self.query = self.getFullQuery()
303+ self.autocomplete_location = (self.user_query_parts, len(self.user_query_parts) - 1)
304+ self.autocomplete_list = []
155305 return self
156306
157307 def getQuery(self):
158308 return ' '.join(self.user_query_parts)
159309
160310 def getFullQuery(self):
161− # get full querry including whitespaces
162− return '{0} {1}'.format(''.join(self.query_parts), self.getQuery()).strip()
311+ """
312+ get full querry including whitespaces
313+ """
314+ return '{0} {1}'.format(' '.join(self.query_parts), self.getQuery()).strip()
315+
316+ def __str__(self):
317+ return self.getFullQuery()
318+
319+ def __repr__(self):
320+ return f"<{self.__class__.__name__} " \
321+ + f"query={self.query!r} " \
322+ + f"disabled_engines={self.disabled_engines!r}\n " \
323+ + f"languages={self.languages!r} " \
324+ + f"timeout_limit={self.timeout_limit!r} "\
325+ + f"external_bang={self.external_bang!r} " \
326+ + f"specific={self.specific!r} " \
327+ + f"enginerefs={self.enginerefs!r}\n " \
328+ + f"autocomplete_list={self.autocomplete_list!r}\n " \
329+ + f"query_parts={self.query_parts!r}\n " \
330+ + f"user_query_parts={self.user_query_parts!r} >"