git.iko.soy/searx
on master help sign in
[enh] record details exception per engine
1d0c368Alexandre Flament
docs/dev/engine_overview.rst+12−10
compare with master
unifiedsplit
@@ -134,16 +134,18 @@ The function ``def request(query, params):`` always returns the ``params``
134variable. Inside searx, the following paramters can be used to specify a search
134variable. Inside searx, the following paramters can be used to specify a search
135request:
135request:
136
136
137−============ =========== =========================================================
137+================== =========== ========================================================================
138−argument type information
138+argument type information
139−============ =========== =========================================================
139+================== =========== ========================================================================
140−url string requested url
140+url string requested url
141−method string HTTP request method
141+method string HTTP request method
142−headers set HTTP header information
142+headers set HTTP header information
143−data set HTTP data information (parsed if ``method != 'GET'``)
143+data set HTTP data information (parsed if ``method != 'GET'``)
144−cookies set HTTP cookies
144+cookies set HTTP cookies
145−verify boolean Performing SSL-Validity check
145+verify boolean Performing SSL-Validity check
146−============ =========== =========================================================
146+max_redirects int maximum redirects, hard limit
147+soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine
148+================== =========== ========================================================================
147
149
148
150
149example code
151example code
searx/engines/__init__.py+2−1
@@ -132,8 +132,9 @@ def load_engine(engine_data):
132 lambda: engine._fetch_supported_languages(get(engine.supported_languages_url)))
132 lambda: engine._fetch_supported_languages(get(engine.supported_languages_url)))
133
133
134 engine.stats = {
134 engine.stats = {
135+ 'sent_search_count': 0, # sent search
136+ 'search_count': 0, # succesful search
135 'result_count': 0,
137 'result_count': 0,
136− 'search_count': 0,
137 'engine_time': 0,
138 'engine_time': 0,
138 'engine_time_count': 0,
139 'engine_time_count': 0,
139 'score_count': 0,
140 'score_count': 0,
searx/exceptions.py+37
@@ -34,8 +34,45 @@ class SearxParameterException(SearxException):
34
34
35
35
36class SearxSettingsException(SearxException):
36class SearxSettingsException(SearxException):
37+ """Error while loading the settings"""
37
38
38 def __init__(self, message, filename):
39 def __init__(self, message, filename):
39 super().__init__(message)
40 super().__init__(message)
40 self.message = message
41 self.message = message
41 self.filename = filename
42 self.filename = filename
43+
44+
45+class SearxEngineException(SearxException):
46+ """Error inside an engine"""
47+
48+
49+class SearxXPathSyntaxException(SearxEngineException):
50+ """Syntax error in a XPATH"""
51+
52+ def __init__(self, xpath_spec, message):
53+ super().__init__(str(xpath_spec) + " " + message)
54+ self.message = message
55+ # str(xpath_spec) to deal with str and XPath instance
56+ self.xpath_str = str(xpath_spec)
57+
58+
59+class SearxEngineResponseException(SearxEngineException):
60+ """Impossible to parse the result of an engine"""
61+
62+
63+class SearxEngineAPIException(SearxEngineResponseException):
64+ """The website has returned an application error"""
65+
66+
67+class SearxEngineCaptchaException(SearxEngineResponseException):
68+ """The website has returned a CAPTCHA"""
69+
70+
71+class SearxEngineXPathException(SearxEngineResponseException):
72+ """Error while getting the result of an XPath expression"""
73+
74+ def __init__(self, xpath_spec, message):
75+ super().__init__(str(xpath_spec) + " " + message)
76+ self.message = message
77+ # str(xpath_spec) to deal with str and XPath instance
78+ self.xpath_str = str(xpath_spec)
searx/metrology/__init__.pynew file

No changes to the content.

searx/metrology/error_recorder.pynew file+142
@@ -0,0 +1,142 @@
1+import typing
2+import inspect
3+import logging
4+from json import JSONDecodeError
5+from urllib.parse import urlparse
6+from requests.exceptions import RequestException
7+from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
8+from searx import logger
9+
10+
11+logging.basicConfig(level=logging.INFO)
12+
13+errors_per_engines = {}
14+
15+
16+class ErrorContext:
17+
18+ __slots__ = 'filename', 'function', 'line_no', 'code', 'exception_classname', 'log_message', 'log_parameters'
19+
20+ def __init__(self, filename, function, line_no, code, exception_classname, log_message, log_parameters):
21+ self.filename = filename
22+ self.function = function
23+ self.line_no = line_no
24+ self.code = code
25+ self.exception_classname = exception_classname
26+ self.log_message = log_message
27+ self.log_parameters = log_parameters
28+
29+ def __eq__(self, o) -> bool:
30+ if not isinstance(o, ErrorContext):
31+ return False
32+ return self.filename == o.filename and self.function == o.function and self.line_no == o.line_no\
33+ and self.code == o.code and self.exception_classname == o.exception_classname\
34+ and self.log_message == o.log_message and self.log_parameters == o.log_parameters
35+
36+ def __hash__(self):
37+ return hash((self.filename, self.function, self.line_no, self.code, self.exception_classname, self.log_message,
38+ self.log_parameters))
39+
40+ def __repr__(self):
41+ return "ErrorContext({!r}, {!r}, {!r}, {!r}, {!r}, {!r})".\
42+ format(self.filename, self.line_no, self.code, self.exception_classname, self.log_message,
43+ self.log_parameters)
44+
45+
46+def add_error_context(engine_name: str, error_context: ErrorContext) -> None:
47+ errors_for_engine = errors_per_engines.setdefault(engine_name, {})
48+ errors_for_engine[error_context] = errors_for_engine.get(error_context, 0) + 1
49+ logger.debug('⚠️ %s: %s', engine_name, str(error_context))
50+
51+
52+def get_trace(traces):
53+ previous_trace = traces[-1]
54+ for trace in reversed(traces):
55+ if trace.filename.endswith('searx/search.py'):
56+ if previous_trace.filename.endswith('searx/poolrequests.py'):
57+ return trace
58+ if previous_trace.filename.endswith('requests/models.py'):
59+ return trace
60+ return previous_trace
61+ previous_trace = trace
62+ return traces[-1]
63+
64+
65+def get_hostname(exc: RequestException) -> typing.Optional[None]:
66+ url = exc.request.url
67+ if url is None and exc.response is not None:
68+ url = exc.response.url
69+ return urlparse(url).netloc
70+
71+
72+def get_request_exception_messages(exc: RequestException)\
73+ -> typing.Tuple[typing.Optional[str], typing.Optional[str], typing.Optional[str]]:
74+ url = None
75+ status_code = None
76+ reason = None
77+ hostname = None
78+ if exc.request is not None:
79+ url = exc.request.url
80+ if url is None and exc.response is not None:
81+ url = exc.response.url
82+ if url is not None:
83+ hostname = str(urlparse(url).netloc)
84+ if exc.response is not None:
85+ status_code = str(exc.response.status_code)
86+ reason = exc.response.reason
87+ return (status_code, reason, hostname)
88+
89+
90+def get_messages(exc, filename) -> typing.Tuple:
91+ if isinstance(exc, JSONDecodeError):
92+ return (exc.msg, )
93+ if isinstance(exc, TypeError):
94+ return (str(exc), )
95+ if isinstance(exc, ValueError) and 'lxml' in filename:
96+ return (str(exc), )
97+ if isinstance(exc, RequestException):
98+ return get_request_exception_messages(exc)
99+ if isinstance(exc, SearxXPathSyntaxException):
100+ return (exc.xpath_str, exc.message)
101+ if isinstance(exc, SearxEngineXPathException):
102+ return (exc.xpath_str, exc.message)
103+ return ()
104+
105+
106+def get_exception_classname(exc: Exception) -> str:
107+ exc_class = exc.__class__
108+ exc_name = exc_class.__qualname__
109+ exc_module = exc_class.__module__
110+ if exc_module is None or exc_module == str.__class__.__module__:
111+ return exc_name
112+ return exc_module + '.' + exc_name
113+
114+
115+def get_error_context(framerecords, exception_classname, log_message, log_parameters) -> ErrorContext:
116+ searx_frame = get_trace(framerecords)
117+ filename = searx_frame.filename
118+ function = searx_frame.function
119+ line_no = searx_frame.lineno
120+ code = searx_frame.code_context[0].strip()
121+ del framerecords
122+ return ErrorContext(filename, function, line_no, code, exception_classname, log_message, log_parameters)
123+
124+
125+def record_exception(engine_name: str, exc: Exception) -> None:
126+ framerecords = inspect.trace()
127+ try:
128+ exception_classname = get_exception_classname(exc)
129+ log_parameters = get_messages(exc, framerecords[-1][1])
130+ error_context = get_error_context(framerecords, exception_classname, None, log_parameters)
131+ add_error_context(engine_name, error_context)
132+ finally:
133+ del framerecords
134+
135+
136+def record_error(engine_name: str, log_message: str, log_parameters: typing.Optional[typing.Tuple] = None) -> None:
137+ framerecords = list(reversed(inspect.stack()[1:]))
138+ try:
139+ error_context = get_error_context(framerecords, None, log_message, log_parameters or ())
140+ add_error_context(engine_name, error_context)
141+ finally:
142+ del framerecords
searx/results.py+9
@@ -4,6 +4,7 @@ from threading import RLock
4from urllib.parse import urlparse, unquote
4from urllib.parse import urlparse, unquote
5from searx import logger
5from searx import logger
6from searx.engines import engines
6from searx.engines import engines
7+from searx.metrology.error_recorder import record_error
7
8
8
9
9CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
10CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
@@ -161,6 +162,7 @@ class ResultContainer:
161
162
162 def extend(self, engine_name, results):
163 def extend(self, engine_name, results):
163 standard_result_count = 0
164 standard_result_count = 0
165+ error_msgs = set()
164 for result in list(results):
166 for result in list(results):
165 result['engine'] = engine_name
167 result['engine'] = engine_name
166 if 'suggestion' in result:
168 if 'suggestion' in result:
@@ -177,14 +179,21 @@ class ResultContainer:
177 # standard result (url, title, content)
179 # standard result (url, title, content)
178 if 'url' in result and not isinstance(result['url'], str):
180 if 'url' in result and not isinstance(result['url'], str):
179 logger.debug('result: invalid URL: %s', str(result))
181 logger.debug('result: invalid URL: %s', str(result))
182+ error_msgs.add('invalid URL')
180 elif 'title' in result and not isinstance(result['title'], str):
183 elif 'title' in result and not isinstance(result['title'], str):
181 logger.debug('result: invalid title: %s', str(result))
184 logger.debug('result: invalid title: %s', str(result))
185+ error_msgs.add('invalid title')
182 elif 'content' in result and not isinstance(result['content'], str):
186 elif 'content' in result and not isinstance(result['content'], str):
183 logger.debug('result: invalid content: %s', str(result))
187 logger.debug('result: invalid content: %s', str(result))
188+ error_msgs.add('invalid content')
184 else:
189 else:
185 self._merge_result(result, standard_result_count + 1)
190 self._merge_result(result, standard_result_count + 1)
186 standard_result_count += 1
191 standard_result_count += 1
187
192
193+ if len(error_msgs) > 0:
194+ for msg in error_msgs:
195+ record_error(engine_name, 'some results are invalids: ' + msg)
196+
188 if engine_name in engines:
197 if engine_name in engines:
189 with RLock():
198 with RLock():
190 engines[engine_name].stats['search_count'] += 1
199 engines[engine_name].stats['search_count'] += 1
searx/search.py+49−5
@@ -20,6 +20,7 @@ import gc
20import threading
20import threading
21from time import time
21from time import time
22from uuid import uuid4
22from uuid import uuid4
23+from urllib.parse import urlparse
23from _thread import start_new_thread
24from _thread import start_new_thread
24
25
25import requests.exceptions
26import requests.exceptions
@@ -31,6 +32,8 @@ from searx.utils import gen_useragent
31from searx.results import ResultContainer
32from searx.results import ResultContainer
32from searx import logger
33from searx import logger
33from searx.plugins import plugins
34from searx.plugins import plugins
35+from searx.exceptions import SearxEngineCaptchaException
36+from searx.metrology.error_recorder import record_exception, record_error
34
37
35
38
36logger = logger.getChild('search')
39logger = logger.getChild('search')
@@ -120,6 +123,14 @@ def send_http_request(engine, request_params):
120 if hasattr(engine, 'proxies'):
123 if hasattr(engine, 'proxies'):
121 request_args['proxies'] = requests_lib.get_proxies(engine.proxies)
124 request_args['proxies'] = requests_lib.get_proxies(engine.proxies)
122
125
126+ # max_redirects
127+ max_redirects = request_params.get('max_redirects')
128+ if max_redirects:
129+ request_args['max_redirects'] = max_redirects
130+
131+ # soft_max_redirects
132+ soft_max_redirects = request_params.get('soft_max_redirects', max_redirects or 0)
133+
123 # specific type of request (GET or POST)
134 # specific type of request (GET or POST)
124 if request_params['method'] == 'GET':
135 if request_params['method'] == 'GET':
125 req = requests_lib.get
136 req = requests_lib.get
@@ -129,7 +140,23 @@ def send_http_request(engine, request_params):
129 request_args['data'] = request_params['data']
140 request_args['data'] = request_params['data']
130
141
131 # send the request
142 # send the request
132− return req(request_params['url'], **request_args)
143+ response = req(request_params['url'], **request_args)
144+
145+ # check HTTP status
146+ response.raise_for_status()
147+
148+ # check soft limit of the redirect count
149+ if len(response.history) > soft_max_redirects:
150+ # unexpected redirect : record an error
151+ # but the engine might still return valid results.
152+ status_code = str(response.status_code or '')
153+ reason = response.reason or ''
154+ hostname = str(urlparse(response.url or '').netloc)
155+ record_error(engine.name,
156+ '{} redirects, maximum: {}'.format(len(response.history), soft_max_redirects),
157+ (status_code, reason, hostname))
158+
159+ return response
133
160
134
161
135def search_one_http_request(engine, query, request_params):
162def search_one_http_request(engine, query, request_params):
@@ -183,8 +210,9 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont
183 # update stats with the total HTTP time
210 # update stats with the total HTTP time
184 engine.stats['page_load_time'] += page_load_time
211 engine.stats['page_load_time'] += page_load_time
185 engine.stats['page_load_count'] += 1
212 engine.stats['page_load_count'] += 1
186−
187 except Exception as e:
213 except Exception as e:
214+ record_exception(engine_name, e)
215+
188 # Timing
216 # Timing
189 engine_time = time() - start_time
217 engine_time = time() - start_time
190 page_load_time = requests_lib.get_time_for_thread()
218 page_load_time = requests_lib.get_time_for_thread()
@@ -195,23 +223,29 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont
195 engine.stats['errors'] += 1
223 engine.stats['errors'] += 1
196
224
197 if (issubclass(e.__class__, requests.exceptions.Timeout)):
225 if (issubclass(e.__class__, requests.exceptions.Timeout)):
198− result_container.add_unresponsive_engine(engine_name, 'timeout')
226+ result_container.add_unresponsive_engine(engine_name, 'HTTP timeout')
199 # requests timeout (connect or read)
227 # requests timeout (connect or read)
200 logger.error("engine {0} : HTTP requests timeout"
228 logger.error("engine {0} : HTTP requests timeout"
201 "(search duration : {1} s, timeout: {2} s) : {3}"
229 "(search duration : {1} s, timeout: {2} s) : {3}"
202 .format(engine_name, engine_time, timeout_limit, e.__class__.__name__))
230 .format(engine_name, engine_time, timeout_limit, e.__class__.__name__))
203 requests_exception = True
231 requests_exception = True
204 elif (issubclass(e.__class__, requests.exceptions.RequestException)):
232 elif (issubclass(e.__class__, requests.exceptions.RequestException)):
205− result_container.add_unresponsive_engine(engine_name, 'request exception')
233+ result_container.add_unresponsive_engine(engine_name, 'HTTP error')
206 # other requests exception
234 # other requests exception
207 logger.exception("engine {0} : requests exception"
235 logger.exception("engine {0} : requests exception"
208 "(search duration : {1} s, timeout: {2} s) : {3}"
236 "(search duration : {1} s, timeout: {2} s) : {3}"
209 .format(engine_name, engine_time, timeout_limit, e))
237 .format(engine_name, engine_time, timeout_limit, e))
210 requests_exception = True
238 requests_exception = True
239+ elif (issubclass(e.__class__, SearxEngineCaptchaException)):
240+ result_container.add_unresponsive_engine(engine_name, 'CAPTCHA required')
241+ logger.exception('engine {0} : CAPTCHA')
211 else:
242 else:
212− result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
243+ result_container.add_unresponsive_engine(engine_name, 'unexpected crash')
213 # others errors
244 # others errors
214 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
245 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
246+ else:
247+ if getattr(threading.current_thread(), '_timeout', False):
248+ record_error(engine_name, 'Timeout')
215
249
216 # suspend or not the engine if there are HTTP errors
250 # suspend or not the engine if there are HTTP errors
217 with threading.RLock():
251 with threading.RLock():
@@ -255,12 +289,17 @@ def search_one_offline_request_safe(engine_name, query, request_params, result_c
255 engine.stats['engine_time_count'] += 1
289 engine.stats['engine_time_count'] += 1
256
290
257 except ValueError as e:
291 except ValueError as e:
292+ record_exception(engine_name, e)
258 record_offline_engine_stats_on_error(engine, result_container, start_time)
293 record_offline_engine_stats_on_error(engine, result_container, start_time)
259 logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))
294 logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))
260 except Exception as e:
295 except Exception as e:
296+ record_exception(engine_name, e)
261 record_offline_engine_stats_on_error(engine, result_container, start_time)
297 record_offline_engine_stats_on_error(engine, result_container, start_time)
262 result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
298 result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
263 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
299 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
300+ else:
301+ if getattr(threading.current_thread(), '_timeout', False):
302+ record_error(engine_name, 'Timeout')
264
303
265
304
266def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
305def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
@@ -278,6 +317,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim
278 args=(engine_name, query, request_params, result_container, start_time, timeout_limit),
317 args=(engine_name, query, request_params, result_container, start_time, timeout_limit),
279 name=search_id,
318 name=search_id,
280 )
319 )
320+ th._timeout = False
281 th._engine_name = engine_name
321 th._engine_name = engine_name
282 th.start()
322 th.start()
283
323
@@ -286,6 +326,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim
286 remaining_time = max(0.0, timeout_limit - (time() - start_time))
326 remaining_time = max(0.0, timeout_limit - (time() - start_time))
287 th.join(remaining_time)
327 th.join(remaining_time)
288 if th.is_alive():
328 if th.is_alive():
329+ th._timeout = True
289 result_container.add_unresponsive_engine(th._engine_name, 'timeout')
330 result_container.add_unresponsive_engine(th._engine_name, 'timeout')
290 logger.warning('engine timeout: {0}'.format(th._engine_name))
331 logger.warning('engine timeout: {0}'.format(th._engine_name))
291
332
@@ -385,6 +426,9 @@ class Search:
385 request_params['category'] = engineref.category
426 request_params['category'] = engineref.category
386 request_params['pageno'] = self.search_query.pageno
427 request_params['pageno'] = self.search_query.pageno
387
428
429+ with threading.RLock():
430+ engine.stats['sent_search_count'] += 1
431+
388 return request_params, engine.timeout
432 return request_params, engine.timeout
389
433
390 # do search-request
434 # do search-request
searx/utils.py+125−23
@@ -10,7 +10,7 @@ from html.parser import HTMLParser
10from urllib.parse import urljoin, urlparse
10from urllib.parse import urljoin, urlparse
11
11
12from lxml import html
12from lxml import html
13−from lxml.etree import XPath, _ElementStringResult, _ElementUnicodeResult
13+from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult
14from babel.core import get_global
14from babel.core import get_global
15
15
16
16
@@ -18,6 +18,7 @@ from searx import settings
18from searx.data import USER_AGENTS
18from searx.data import USER_AGENTS
19from searx.version import VERSION_STRING
19from searx.version import VERSION_STRING
20from searx.languages import language_codes
20from searx.languages import language_codes
21+from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
21from searx import logger
22from searx import logger
22
23
23
24
@@ -33,6 +34,13 @@ xpath_cache = dict()
33lang_to_lc_cache = dict()
34lang_to_lc_cache = dict()
34
35
35
36
37+class NotSetClass:
38+ pass
39+
40+
41+NOTSET = NotSetClass()
42+
43+
36def searx_useragent():
44def searx_useragent():
37 """Return the searx User Agent"""
45 """Return the searx User Agent"""
38 return 'searx/{searx_version} {suffix}'.format(
46 return 'searx/{searx_version} {suffix}'.format(
@@ -125,7 +133,7 @@ def html_to_text(html_str):
125 return s.get_text()
133 return s.get_text()
126
134
127
135
128−def extract_text(xpath_results):
136+def extract_text(xpath_results, allow_none=False):
129 """Extract text from a lxml result
137 """Extract text from a lxml result
130
138
131 * if xpath_results is list, extract the text from each result and concat the list
139 * if xpath_results is list, extract the text from each result and concat the list
@@ -133,22 +141,27 @@ def extract_text(xpath_results):
133 ( text_content() method from lxml )
141 ( text_content() method from lxml )
134 * if xpath_results is a string element, then it's already done
142 * if xpath_results is a string element, then it's already done
135 """
143 """
136− if type(xpath_results) == list:
144+ if isinstance(xpath_results, list):
137 # it's list of result : concat everything using recursive call
145 # it's list of result : concat everything using recursive call
138 result = ''
146 result = ''
139 for e in xpath_results:
147 for e in xpath_results:
140 result = result + extract_text(e)
148 result = result + extract_text(e)
141 return result.strip()
149 return result.strip()
142− elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]:
150+ elif isinstance(xpath_results, ElementBase):
143− # it's a string
144− return ''.join(xpath_results)
145− else:
146 # it's a element
151 # it's a element
147 text = html.tostring(
152 text = html.tostring(
148 xpath_results, encoding='unicode', method='text', with_tail=False
153 xpath_results, encoding='unicode', method='text', with_tail=False
149 )
154 )
150 text = text.strip().replace('\n', ' ')
155 text = text.strip().replace('\n', ' ')
151 return ' '.join(text.split())
156 return ' '.join(text.split())
157+ elif isinstance(xpath_results, (_ElementStringResult, _ElementUnicodeResult, str, Number, bool)):
158+ return str(xpath_results)
159+ elif xpath_results is None and allow_none:
160+ return None
161+ elif xpath_results is None and not allow_none:
162+ raise ValueError('extract_text(None, allow_none=False)')
163+ else:
164+ raise ValueError('unsupported type')
152
165
153
166
154def normalize_url(url, base_url):
167def normalize_url(url, base_url):
@@ -170,7 +183,7 @@ def normalize_url(url, base_url):
170 >>> normalize_url('', 'https://example.com')
183 >>> normalize_url('', 'https://example.com')
171 'https://example.com/'
184 'https://example.com/'
172 >>> normalize_url('/test', '/path')
185 >>> normalize_url('/test', '/path')
173− raise Exception
186+ raise ValueError
174
187
175 Raises:
188 Raises:
176 * lxml.etree.ParserError
189 * lxml.etree.ParserError
@@ -194,7 +207,7 @@ def normalize_url(url, base_url):
194
207
195 # add a / at this end of the url if there is no path
208 # add a / at this end of the url if there is no path
196 if not parsed_url.netloc:
209 if not parsed_url.netloc:
197− raise Exception('Cannot parse url')
210+ raise ValueError('Cannot parse url')
198 if not parsed_url.path:
211 if not parsed_url.path:
199 url += '/'
212 url += '/'
200
213
@@ -224,17 +237,17 @@ def extract_url(xpath_results, base_url):
224 >>> f('', 'https://example.com')
237 >>> f('', 'https://example.com')
225 raise lxml.etree.ParserError
238 raise lxml.etree.ParserError
226 >>> searx.utils.extract_url([], 'https://example.com')
239 >>> searx.utils.extract_url([], 'https://example.com')
227− raise Exception
240+ raise ValueError
228
241
229 Raises:
242 Raises:
230− * Exception
243+ * ValueError
231 * lxml.etree.ParserError
244 * lxml.etree.ParserError
232
245
233 Returns:
246 Returns:
234 * str: normalized URL
247 * str: normalized URL
235 """
248 """
236 if xpath_results == []:
249 if xpath_results == []:
237− raise Exception('Empty url resultset')
250+ raise ValueError('Empty url resultset')
238
251
239 url = extract_text(xpath_results)
252 url = extract_text(xpath_results)
240 return normalize_url(url, base_url)
253 return normalize_url(url, base_url)
@@ -258,7 +271,6 @@ def dict_subset(d, properties):
258
271
259def list_get(a_list, index, default=None):
272def list_get(a_list, index, default=None):
260 """Get element in list or default value
273 """Get element in list or default value
261−
262 Examples:
274 Examples:
263 >>> list_get(['A', 'B', 'C'], 0)
275 >>> list_get(['A', 'B', 'C'], 0)
264 'A'
276 'A'
@@ -310,7 +322,7 @@ def get_torrent_size(filesize, filesize_multiplier):
310 filesize = int(filesize * 1000 * 1000)
322 filesize = int(filesize * 1000 * 1000)
311 elif filesize_multiplier == 'KiB':
323 elif filesize_multiplier == 'KiB':
312 filesize = int(filesize * 1000)
324 filesize = int(filesize * 1000)
313− except:
325+ except ValueError:
314 filesize = None
326 filesize = None
315
327
316 return filesize
328 return filesize
@@ -506,20 +518,110 @@ def get_engine_from_settings(name):
506 return {}
518 return {}
507
519
508
520
509−def get_xpath(xpath_str):
521+def get_xpath(xpath_spec):
510 """Return cached compiled XPath
522 """Return cached compiled XPath
511
523
512 There is no thread lock.
524 There is no thread lock.
513 Worst case scenario, xpath_str is compiled more than one time.
525 Worst case scenario, xpath_str is compiled more than one time.
526+
527+ Args:
528+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
529+
530+ Returns:
531+ * result (bool, float, list, str): Results.
532+
533+ Raises:
534+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
535+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
536+ """
537+ if isinstance(xpath_spec, str):
538+ result = xpath_cache.get(xpath_spec, None)
539+ if result is None:
540+ try:
541+ result = XPath(xpath_spec)
542+ except XPathSyntaxError as e:
543+ raise SearxXPathSyntaxException(xpath_spec, str(e.msg))
544+ xpath_cache[xpath_spec] = result
545+ return result
546+
547+ if isinstance(xpath_spec, XPath):
548+ return xpath_spec
549+
550+ raise TypeError('xpath_spec must be either a str or a lxml.etree.XPath')
551+
552+
553+def eval_xpath(element, xpath_spec):
554+ """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all.
555+ See https://lxml.de/xpathxslt.html#xpath-return-values
556+
557+ Args:
558+ * element (ElementBase): [description]
559+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
560+
561+ Returns:
562+ * result (bool, float, list, str): Results.
563+
564+ Raises:
565+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
566+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
567+ * SearxEngineXPathException: Raise when the XPath can't be evaluated.
514 """
568 """
515− result = xpath_cache.get(xpath_str, None)
569+ xpath = get_xpath(xpath_spec)
516− if result is None:
570+ try:
517− result = XPath(xpath_str)
571+ return xpath(element)
518− xpath_cache[xpath_str] = result
572+ except XPathError as e:
573+ arg = ' '.join([str(i) for i in e.args])
574+ raise SearxEngineXPathException(xpath_spec, arg)
575+
576+
577+def eval_xpath_list(element, xpath_spec, min_len=None):
578+ """Same as eval_xpath, check if the result is a list
579+
580+ Args:
581+ * element (ElementBase): [description]
582+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath
583+ * min_len (int, optional): [description]. Defaults to None.
584+
585+ Raises:
586+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
587+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
588+ * SearxEngineXPathException: raise if the result is not a list
589+
590+ Returns:
591+ * result (bool, float, list, str): Results.
592+ """
593+ result = eval_xpath(element, xpath_spec)
594+ if not isinstance(result, list):
595+ raise SearxEngineXPathException(xpath_spec, 'the result is not a list')
596+ if min_len is not None and min_len > len(result):
597+ raise SearxEngineXPathException(xpath_spec, 'len(xpath_str) < ' + str(min_len))
519 return result
598 return result
520
599
521
600
522−def eval_xpath(element, xpath_str):
601+def eval_xpath_getindex(elements, xpath_spec, index, default=NOTSET):
523− """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all."""
602+ """Call eval_xpath_list then get one element using the index parameter.
524− xpath = get_xpath(xpath_str)
603+ If the index does not exist, either aise an exception is default is not set,
525− return xpath(element)
604+ other return the default value (can be None).
605+
606+ Args:
607+ * elements (ElementBase): lxml element to apply the xpath.
608+ * xpath_spec (str|lxml.etree.XPath): XPath as a str or lxml.etree.XPath.
609+ * index (int): index to get
610+ * default (Object, optional): Defaults if index doesn't exist.
611+
612+ Raises:
613+ * TypeError: Raise when xpath_spec is neither a str nor a lxml.etree.XPath
614+ * SearxXPathSyntaxException: Raise when there is a syntax error in the XPath
615+ * SearxEngineXPathException: if the index is not found. Also see eval_xpath.
616+
617+ Returns:
618+ * result (bool, float, list, str): Results.
619+ """
620+ result = eval_xpath_list(elements, xpath_spec)
621+ if index >= -len(result) and index < len(result):
622+ return result[index]
623+ if default == NOTSET:
624+ # raise an SearxEngineXPathException instead of IndexError
625+ # to record xpath_spec
626+ raise SearxEngineXPathException(xpath_spec, 'index ' + str(index) + ' not found')
627+ return default
searx/webapp.py+29
@@ -79,6 +79,7 @@ from searx.plugins.oa_doi_rewrite import get_doi_resolver
79from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
79from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
80from searx.answerers import answerers
80from searx.answerers import answerers
81from searx.poolrequests import get_global_proxies
81from searx.poolrequests import get_global_proxies
82+from searx.metrology.error_recorder import errors_per_engines
82
83
83
84
84# serve pages with HTTP/1.1
85# serve pages with HTTP/1.1
@@ -943,6 +944,34 @@ def stats():
943 )
944 )
944
945
945
946
947+@app.route('/stats/errors', methods=['GET'])
948+def stats_errors():
949+ result = {}
950+ engine_names = list(errors_per_engines.keys())
951+ engine_names.sort()
952+ for engine_name in engine_names:
953+ error_stats = errors_per_engines[engine_name]
954+ sent_search_count = max(engines[engine_name].stats['sent_search_count'], 1)
955+ sorted_context_count_list = sorted(error_stats.items(), key=lambda context_count: context_count[1])
956+ r = []
957+ percentage_sum = 0
958+ for context, count in sorted_context_count_list:
959+ percentage = round(20 * count / sent_search_count) * 5
960+ percentage_sum += percentage
961+ r.append({
962+ 'filename': context.filename,
963+ 'function': context.function,
964+ 'line_no': context.line_no,
965+ 'code': context.code,
966+ 'exception_classname': context.exception_classname,
967+ 'log_message': context.log_message,
968+ 'log_parameters': context.log_parameters,
969+ 'percentage': percentage,
970+ })
971+ result[engine_name] = sorted(r, reverse=True, key=lambda d: d['percentage'])
972+ return jsonify(result)
973+
974+
946@app.route('/robots.txt', methods=['GET'])
975@app.route('/robots.txt', methods=['GET'])
947def robots():
976def robots():
948 return Response("""User-agent: *
977 return Response("""User-agent: *
tests/unit/test_utils.py+90
@@ -3,6 +3,7 @@ import lxml.etree
3from lxml import html
3from lxml import html
4
4
5from searx.testing import SearxTestCase
5from searx.testing import SearxTestCase
6+from searx.exceptions import SearxXPathSyntaxException, SearxEngineXPathException
6from searx import utils
7from searx import utils
7
8
8
9
@@ -57,8 +58,16 @@ class TestUtils(SearxTestCase):
57 dom = html.fromstring(html_str)
58 dom = html.fromstring(html_str)
58 self.assertEqual(utils.extract_text(dom), 'Test text')
59 self.assertEqual(utils.extract_text(dom), 'Test text')
59 self.assertEqual(utils.extract_text(dom.xpath('//span')), 'Test text')
60 self.assertEqual(utils.extract_text(dom.xpath('//span')), 'Test text')
61+ self.assertEqual(utils.extract_text(dom.xpath('//span/text()')), 'Test text')
62+ self.assertEqual(utils.extract_text(dom.xpath('count(//span)')), '3.0')
63+ self.assertEqual(utils.extract_text(dom.xpath('boolean(//span)')), 'True')
60 self.assertEqual(utils.extract_text(dom.xpath('//img/@src')), 'test.jpg')
64 self.assertEqual(utils.extract_text(dom.xpath('//img/@src')), 'test.jpg')
61 self.assertEqual(utils.extract_text(dom.xpath('//unexistingtag')), '')
65 self.assertEqual(utils.extract_text(dom.xpath('//unexistingtag')), '')
66+ self.assertEqual(utils.extract_text(None, allow_none=True), None)
67+ with self.assertRaises(ValueError):
68+ utils.extract_text(None)
69+ with self.assertRaises(ValueError):
70+ utils.extract_text({})
62
71
63 def test_extract_url(self):
72 def test_extract_url(self):
64 def f(html_str, search_url):
73 def f(html_str, search_url):
@@ -136,3 +145,84 @@ class TestHTMLTextExtractor(SearxTestCase):
136 text = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
145 text = '<p><b>Lorem ipsum</i>dolor sit amet</p>'
137 with self.assertRaises(utils.HTMLTextExtractorException):
146 with self.assertRaises(utils.HTMLTextExtractorException):
138 self.html_text_extractor.feed(text)
147 self.html_text_extractor.feed(text)
148+
149+
150+class TestXPathUtils(SearxTestCase):
151+
152+ TEST_DOC = """<ul>
153+ <li>Text in <b>bold</b> and <i>italic</i> </li>
154+ <li>Another <b>text</b> <img src="data:image/gif;base64,R0lGODlhAQABAIAAAAUEBAAAACwAAAAAAQABAAACAkQBADs="></li>
155+ </ul>"""
156+
157+ def test_get_xpath_cache(self):
158+ xp1 = utils.get_xpath('//a')
159+ xp2 = utils.get_xpath('//div')
160+ xp3 = utils.get_xpath('//a')
161+
162+ self.assertEqual(id(xp1), id(xp3))
163+ self.assertNotEqual(id(xp1), id(xp2))
164+
165+ def test_get_xpath_type(self):
166+ utils.get_xpath(lxml.etree.XPath('//a'))
167+
168+ with self.assertRaises(TypeError):
169+ utils.get_xpath([])
170+
171+ def test_get_xpath_invalid(self):
172+ invalid_xpath = '//a[0].text'
173+ with self.assertRaises(SearxXPathSyntaxException) as context:
174+ utils.get_xpath(invalid_xpath)
175+
176+ self.assertEqual(context.exception.message, 'Invalid expression')
177+ self.assertEqual(context.exception.xpath_str, invalid_xpath)
178+
179+ def test_eval_xpath_unregistered_function(self):
180+ doc = html.fromstring(TestXPathUtils.TEST_DOC)
181+
182+ invalid_function_xpath = 'int(//a)'
183+ with self.assertRaises(SearxEngineXPathException) as context:
184+ utils.eval_xpath(doc, invalid_function_xpath)
185+
186+ self.assertEqual(context.exception.message, 'Unregistered function')
187+ self.assertEqual(context.exception.xpath_str, invalid_function_xpath)
188+
189+ def test_eval_xpath(self):
190+ doc = html.fromstring(TestXPathUtils.TEST_DOC)
191+
192+ self.assertEqual(utils.eval_xpath(doc, '//p'), [])
193+ self.assertEqual(utils.eval_xpath(doc, '//i/text()'), ['italic'])
194+ self.assertEqual(utils.eval_xpath(doc, 'count(//i)'), 1.0)
195+
196+ def test_eval_xpath_list(self):
197+ doc = html.fromstring(TestXPathUtils.TEST_DOC)
198+
199+ # check a not empty list
200+ self.assertEqual(utils.eval_xpath_list(doc, '//i/text()'), ['italic'])
201+
202+ # check min_len parameter
203+ with self.assertRaises(SearxEngineXPathException) as context:
204+ utils.eval_xpath_list(doc, '//p', min_len=1)
205+ self.assertEqual(context.exception.message, 'len(xpath_str) < 1')
206+ self.assertEqual(context.exception.xpath_str, '//p')
207+
208+ def test_eval_xpath_getindex(self):
209+ doc = html.fromstring(TestXPathUtils.TEST_DOC)
210+
211+ # check index 0
212+ self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 0), 'italic')
213+
214+ # default is 'something'
215+ self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 1, default='something'), 'something')
216+
217+ # default is None
218+ self.assertEqual(utils.eval_xpath_getindex(doc, '//i/text()', 1, default=None), None)
219+
220+ # index not found
221+ with self.assertRaises(SearxEngineXPathException) as context:
222+ utils.eval_xpath_getindex(doc, '//i/text()', 1)
223+ self.assertEqual(context.exception.message, 'index 1 not found')
224+
225+ # not a list
226+ with self.assertRaises(SearxEngineXPathException) as context:
227+ utils.eval_xpath_getindex(doc, 'count(//i)', 1)
228+ self.assertEqual(context.exception.message, 'the result is not a list')