@@ -0,0 +1,76 @@
1+"""
2+ Duden
3+ @website https://www.duden.de
4+ @provide-api no
5+ @using-api no
6+ @results HTML (using search portal)
7+ @stable no (HTML can change)
8+ @parse url, title, content
9+"""
10+
11+from lxml import html, etree
12+import re
13+from searx.engines.xpath import extract_text
14+from searx.url_utils import quote
15+from searx import logger
16+
17+categories = ['general']
18+paging = True
19+language_support = False
20+
21+# search-url
22+base_url = 'https://www.duden.de/'
23+search_url = base_url + 'suchen/dudenonline/{query}?page={offset}'
24+
25+
26+def request(query, params):
27+ '''pre-request callback
28+ params<dict>:
29+ method : POST/GET
30+ headers : {}
31+ data : {} # if method == POST
32+ url : ''
33+ category: 'search category'
34+ pageno : 1 # number of the requested page
35+ '''
36+
37+ offset = (params['pageno'] - 1)
38+ params['url'] = search_url.format(offset=offset, query=quote(query))
39+ return params
40+
41+
42+def response(resp):
43+ '''post-response callback
44+ resp: requests response object
45+ '''
46+ results = []
47+
48+ dom = html.fromstring(resp.text)
49+
50+ try:
51+ number_of_results_string = re.sub('[^0-9]', '', dom.xpath(
52+ '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]
53+ )
54+
55+ results.append({'number_of_results': int(number_of_results_string)})
56+
57+ except:
58+ logger.debug("Couldn't read number of results.")
59+ pass
60+
61+ for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'):
62+ try:
63+ logger.debug("running for %s" % str(result))
64+ link = result.xpath('.//h2/a')[0]
65+ url = link.attrib.get('href')
66+ title = result.xpath('string(.//h2/a)')
67+ content = extract_text(result.xpath('.//p'))
68+ # append result
69+ results.append({'url': url,
70+ 'title': title,
71+ 'content': content})
72+ except:
73+ logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True))
74+ continue
75+
76+ return results