@@ -8,11 +8,10 @@
88 @parse url, title, content
99"""
1010
11−from lxml import html, etree
1211import re
1312from urllib.parse import quote, urljoin
14−from searx.utils import extract_text, eval_xpath
15−from searx import logger
13+from lxml import html
14+from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
1615
1716categories = ['general']
1817paging = True
@@ -40,6 +39,9 @@ def request(query, params):
4039 params['url'] = search_url_fmt.format(query=quote(query))
4140 else:
4241 params['url'] = search_url.format(offset=offset, query=quote(query))
42+ # after the last page of results, spelling corrections are returned after a HTTP redirect
43+ # whatever the page number is
44+ params['soft_max_redirects'] = 1
4345 return params
4446
4547
@@ -51,28 +53,21 @@ def response(resp):
5153
5254 dom = html.fromstring(resp.text)
5355
54− try:
55− number_of_results_string =\
56− re.sub('[^0-9]', '',
57− eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0])
58−
56+ number_of_results_element =\
57+ eval_xpath_getindex(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()',
58+ 0, default=None)
59+ if number_of_results_element is not None:
60+ number_of_results_string = re.sub('[^0-9]', '', number_of_results_element)
5961 results.append({'number_of_results': int(number_of_results_string)})
6062
61− except:
62− logger.debug("Couldn't read number of results.")
63−
64− for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'):
65− try:
66− url = eval_xpath(result, './/h2/a')[0].get('href')
67− url = urljoin(base_url, url)
68− title = eval_xpath(result, 'string(.//h2/a)').strip()
69− content = extract_text(eval_xpath(result, './/p'))
70− # append result
71− results.append({'url': url,
72− 'title': title,
73− 'content': content})
74− except:
75− logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True))
76− continue
63+ for result in eval_xpath_list(dom, '//section[not(contains(@class, "essay"))]'):
64+ url = eval_xpath_getindex(result, './/h2/a', 0).get('href')
65+ url = urljoin(base_url, url)
66+ title = eval_xpath(result, 'string(.//h2/a)').strip()
67+ content = extract_text(eval_xpath(result, './/p'))
68+ # append result
69+ results.append({'url': url,
70+ 'title': title,
71+ 'content': content})
7772
7873 return results