git.iko.soy/searx
on master help sign in
[fix] fix duden engine (#1594)
commit3b1122c5fa1bbac49f6e0eb9ff69efb8dca996ea authorrachmadani haryono <rachmadaniHaryono@users.noreply.github.com> dateThu 25 Jul 2019, 14:17 (7 years ago) parent87baa74
2 files changed+28−19
compare with master
unifiedsplit
searx/engines/duden.py+11−8 tests/unit/engines/test_duden.py+17−11
searx/engines/duden.py+11−8
@@ -11,7 +11,7 @@
1111from lxml import html, etree
1212import re
1313from searx.engines.xpath import extract_text
14−from searx.url_utils import quote
14+from searx.url_utils import quote, urljoin
1515from searx import logger
1616
1717categories = ['general']
@@ -20,7 +20,7 @@ language_support = False
2020
2121# search-url
2222base_url = 'https://www.duden.de/'
23−search_url = base_url + 'suchen/dudenonline/{query}?page={offset}'
23+search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}'
2424
2525
2626def request(query, params):
@@ -35,7 +35,11 @@ def request(query, params):
3535 '''
3636
3737 offset = (params['pageno'] - 1)
38− params['url'] = search_url.format(offset=offset, query=quote(query))
38+ if offset == 0:
39+ search_url_fmt = base_url + 'suchen/dudenonline/{query}'
40+ params['url'] = search_url_fmt.format(query=quote(query))
41+ else:
42+ params['url'] = search_url.format(offset=offset, query=quote(query))
3943 return params
4044
4145
@@ -58,12 +62,11 @@ def response(resp):
5862 logger.debug("Couldn't read number of results.")
5963 pass
6064
61− for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'):
65+ for result in dom.xpath('//section[not(contains(@class, "essay"))]'):
6266 try:
63− logger.debug("running for %s" % str(result))
64− link = result.xpath('.//h2/a')[0]
65− url = link.attrib.get('href')
66− title = result.xpath('string(.//h2/a)')
67+ url = result.xpath('.//h2/a')[0].get('href')
68+ url = urljoin(base_url, url)
69+ title = result.xpath('string(.//h2/a)').strip()
6770 content = extract_text(result.xpath('.//p'))
6871 # append result
6972 results.append({'url': url,
tests/unit/engines/test_duden.py+17−11
@@ -10,24 +10,30 @@ class TestDudenEngine(SearxTestCase):
1010 def test_request(self):
1111 query = 'Haus'
1212 dic = defaultdict(dict)
13− dic['pageno'] = 1
14− params = duden.request(query, dic)
15− self.assertTrue('url' in params)
16− self.assertTrue(query in params['url'])
17− self.assertTrue('duden.de' in params['url'])
13+ data = [
14+ [1, 'https://www.duden.de/suchen/dudenonline/Haus'],
15+ [2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1']
16+ ]
17+ for page_no, exp_res in data:
18+ dic['pageno'] = page_no
19+ params = duden.request(query, dic)
20+ self.assertTrue('url' in params)
21+ self.assertTrue(query in params['url'])
22+ self.assertTrue('duden.de' in params['url'])
23+ self.assertEqual(params['url'], exp_res)
1824
1925 def test_response(self):
2026 resp = mock.Mock(text='<html></html>')
2127 self.assertEqual(duden.response(resp), [])
2228
2329 html = """
24− <section class="wide">
25− <h2><a href="https://this.is.the.url/" class="hidden-link"><strong>This is the title</strong> also here</a></h2>
26− <p>This is the <strong>content</strong></p>
27− <a href="https://this.is.the.url/">Zum vollst&auml;ndigen Artikel</a>
30+ <section class="vignette">
31+ <h2"> <a href="/rechtschreibung/Haus">
32+ <strong>This is the title also here</strong>
33+ </a> </h2>
34+ <p>This is the content</p>
2835 </section>
2936 """
30−
3137 resp = mock.Mock(text=html)
3238 results = duden.response(resp)
3339
@@ -36,6 +42,6 @@ class TestDudenEngine(SearxTestCase):
3642
3743 # testing result (dictionary entry)
3844 r = results[0]
39− self.assertEqual(r['url'], 'https://this.is.the.url/')
45+ self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus')
4046 self.assertEqual(r['title'], 'This is the title also here')
4147 self.assertEqual(r['content'], 'This is the content')