@@ -20,9 +20,10 @@ Definitions`_.
2020
2121from urllib.parse import urlencode, urlparse
2222from lxml import html
23−from flask_babel import gettext
2423from searx import logger
25−from searx.utils import match_language, extract_text, eval_xpath
24+from searx.utils import match_language, extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
25+from searx.exceptions import SearxEngineCaptchaException
26+
2627
2728logger = logger.getChild('google engine')
2829
@@ -131,14 +132,6 @@ suggestion_xpath = '//div[contains(@class, "card-section")]//a'
131132spelling_suggestion_xpath = '//div[@class="med"]/p/a'
132133
133134
134−def extract_text_from_dom(result, xpath):
135− """returns extract_text on the first result selected by the xpath or None"""
136− r = eval_xpath(result, xpath)
137− if len(r) > 0:
138− return extract_text(r[0])
139− return None
140−
141−
142135def get_lang_country(params, lang_list, custom_aliases):
143136 """Returns a tuple with *langauage* on its first and *country* on its second
144137 position."""
@@ -210,10 +203,10 @@ def response(resp):
210203 # detect google sorry
211204 resp_url = urlparse(resp.url)
212205 if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':
213− raise RuntimeWarning('sorry.google.com')
206+ raise SearxEngineCaptchaException()
214207
215208 if resp_url.path.startswith('/sorry'):
216− raise RuntimeWarning(gettext('CAPTCHA required'))
209+ raise SearxEngineCaptchaException()
217210
218211 # which subdomain ?
219212 # subdomain = resp.search_params.get('google_subdomain')
@@ -229,18 +222,17 @@ def response(resp):
229222 logger.debug("did not found 'answer'")
230223
231224 # results --> number_of_results
232− try:
233− _txt = eval_xpath(dom, '//div[@id="result-stats"]//text()')[0]
234− _digit = ''.join([n for n in _txt if n.isdigit()])
235− number_of_results = int(_digit)
236− results.append({'number_of_results': number_of_results})
237−
238− except Exception as e: # pylint: disable=broad-except
239− logger.debug("did not 'number_of_results'")
240− logger.error(e, exc_info=True)
225+ try:
226+ _txt = eval_xpath_getindex(dom, '//div[@id="result-stats"]//text()', 0)
227+ _digit = ''.join([n for n in _txt if n.isdigit()])
228+ number_of_results = int(_digit)
229+ results.append({'number_of_results': number_of_results})
230+ except Exception as e: # pylint: disable=broad-except
231+ logger.debug("did not 'number_of_results'")
232+ logger.error(e, exc_info=True)
241233
242234 # parse results
243− for result in eval_xpath(dom, results_xpath):
235+ for result in eval_xpath_list(dom, results_xpath):
244236
245237 # google *sections*
246238 if extract_text(eval_xpath(result, g_section_with_header)):
@@ -248,14 +240,14 @@ def response(resp):
248240 continue
249241
250242 try:
251− title_tag = eval_xpath(result, title_xpath)
252− if not title_tag:
243+ title_tag = eval_xpath_getindex(result, title_xpath, 0, default=None)
244+ if title_tag is None:
253245 # this not one of the common google results *section*
254246 logger.debug('ingoring <div class="g" ../> section: missing title')
255247 continue
256− title = extract_text(title_tag[0])
257− url = eval_xpath(result, href_xpath)[0]
258− content = extract_text_from_dom(result, content_xpath)
248+ title = extract_text(title_tag)
249+ url = eval_xpath_getindex(result, href_xpath, 0)
250+ content = extract_text(eval_xpath_getindex(result, content_xpath, 0, default=None), allow_none=True)
259251 results.append({
260252 'url': url,
261253 'title': title,
@@ -270,11 +262,11 @@ def response(resp):
270262 continue
271263
272264 # parse suggestion
273− for suggestion in eval_xpath(dom, suggestion_xpath):
265+ for suggestion in eval_xpath_list(dom, suggestion_xpath):
274266 # append suggestion
275267 results.append({'suggestion': extract_text(suggestion)})
276268
277− for correction in eval_xpath(dom, spelling_suggestion_xpath):
269+ for correction in eval_xpath_list(dom, spelling_suggestion_xpath):
278270 results.append({'correction': extract_text(correction)})
279271
280272 # return results
@@ -286,7 +278,7 @@ def _fetch_supported_languages(resp):
286278 ret_val = {}
287279 dom = html.fromstring(resp.text)
288280
289− radio_buttons = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')
281+ radio_buttons = eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]')
290282
291283 for x in radio_buttons:
292284 name = x.get("data-name")