3b1122c · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[fix] fix duden engine (#1594)

3b1122crachmadani haryono7 years ago

searx/engines/duden.py+11−8

compare with master

unifiedsplit

@@ -11,7 +11,7 @@

11from lxml import html, etree

12import re

13from searx.engines.xpath import extract_text

14−from searx.url_utils import quote

14+from searx.url_utils import quote, urljoin

15from searx import logger

17categories = ['general']

@@ -20,7 +20,7 @@ language_support = False

21# search-url

22base_url = 'https://www.duden.de/'

23−search_url = base_url + 'suchen/dudenonline/{query}?page={offset}'

23+search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}'

26def request(query, params):

@@ -35,7 +35,11 @@ def request(query, params):

35 '''

37 offset = (params['pageno'] - 1)

38+ if offset == 0:

39+ search_url_fmt = base_url + 'suchen/dudenonline/{query}'

40+ params['url'] = search_url_fmt.format(query=quote(query))

41+ else:

38− params['url'] = search_url.format(offset=offset, query=quote(query))

42+ params['url'] = search_url.format(offset=offset, query=quote(query))

39 return params

43 return params

@@ -58,12 +62,11 @@ def response(resp):

58 logger.debug("Couldn't read number of results.")

62 logger.debug("Couldn't read number of results.")

59 pass

63 pass

61− for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'):

65+ for result in dom.xpath('//section[not(contains(@class, "essay"))]'):

62 try:

66 try:

63− logger.debug("running for %s" % str(result))

64− link = result.xpath('.//h2/a')[0]

67+ url = result.xpath('.//h2/a')[0].get('href')

65− url = link.attrib.get('href')

68+ url = urljoin(base_url, url)

66− title = result.xpath('string(.//h2/a)')

69+ title = result.xpath('string(.//h2/a)').strip()

67 content = extract_text(result.xpath('.//p'))

70 content = extract_text(result.xpath('.//p'))

68 # append result

71 # append result

69 results.append({'url': url,

72 results.append({'url': url,

tests/unit/engines/test_duden.py+17−11

@@ -10,24 +10,30 @@ class TestDudenEngine(SearxTestCase):

10 def test_request(self):

11 query = 'Haus'

12 dic = defaultdict(dict)

13+ data = [

14+ [1, 'https://www.duden.de/suchen/dudenonline/Haus'],

15+ [2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1']

16+ ]

17+ for page_no, exp_res in data:

13− dic['pageno'] = 1

18+ dic['pageno'] = page_no

14− params = duden.request(query, dic)

19+ params = duden.request(query, dic)

15− self.assertTrue('url' in params)

20+ self.assertTrue('url' in params)

16− self.assertTrue(query in params['url'])

21+ self.assertTrue(query in params['url'])

17− self.assertTrue('duden.de' in params['url'])

22+ self.assertTrue('duden.de' in params['url'])

23+ self.assertEqual(params['url'], exp_res)

19 def test_response(self):

25 def test_response(self):

20 resp = mock.Mock(text='<html></html>')

26 resp = mock.Mock(text='<html></html>')

21 self.assertEqual(duden.response(resp), [])

27 self.assertEqual(duden.response(resp), [])

23 html = """

29 html = """

24− <section class="wide">

30+ <section class="vignette">

31+ <h2"> <a href="/rechtschreibung/Haus">

25− <h2><a href="https://this.is.the.url/" class="hidden-link"><strong>This is the title</strong> also here</a></h2>

32+ <strong>This is the title also here</strong>

33+ </a> </h2>

26− <p>This is the <strong>content</strong></p>

34+ <p>This is the content</p>

27− <a href="https://this.is.the.url/">Zum vollständigen Artikel</a>

28 </section>

35 </section>

29 """

36 """

30−

31 resp = mock.Mock(text=html)

37 resp = mock.Mock(text=html)

32 results = duden.response(resp)

38 results = duden.response(resp)

@@ -36,6 +42,6 @@ class TestDudenEngine(SearxTestCase):

37 # testing result (dictionary entry)

43 # testing result (dictionary entry)

38 r = results[0]

44 r = results[0]

39− self.assertEqual(r['url'], 'https://this.is.the.url/')

45+ self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus')

40 self.assertEqual(r['title'], 'This is the title also here')

46 self.assertEqual(r['title'], 'This is the title also here')

41 self.assertEqual(r['content'], 'This is the content')

47 self.assertEqual(r['content'], 'This is the content')