c2da901 · searx · git.iko.soy

searx/answerers/random/answerer.py+21−3

compare with master

unifiedsplit

@@ -1,6 +1,8 @@

1+import hashlib

1import random

2import random

2import string

3import string

3import sys

4import sys

5+import uuid

4from flask_babel import gettext

6from flask_babel import gettext

5

7

6# required answerer attribute

8# required answerer attribute

@@ -16,9 +18,13 @@ else:

16 random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase

18 random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase

17

19

18

20

21+def random_characters():

22+ return [random.choice(random_string_letters)

23+ for _ in range(random.randint(8, 32))]

24+

25+

19def random_string():

26def random_string():

20− return u''.join(random.choice(random_string_letters)

27+ return u''.join(random_characters())

21− for _ in range(random.randint(8, 32)))

22

28

23

29

24def random_float():

30def random_float():

@@ -29,9 +35,21 @@ def random_int():

29 return unicode(random.randint(-random_int_max, random_int_max))

35 return unicode(random.randint(-random_int_max, random_int_max))

30

36

31

37

38+def random_sha256():

39+ m = hashlib.sha256()

40+ m.update(b''.join(random_characters()))

41+ return unicode(m.hexdigest())

42+

43+

44+def random_uuid():

45+ return unicode(uuid.uuid4())

46+

47+

32random_types = {b'string': random_string,

48random_types = {b'string': random_string,

33 b'int': random_int,

49 b'int': random_int,

34− b'float': random_float}

50+ b'float': random_float,

51+ b'sha256': random_sha256,

52+ b'uuid': random_uuid}

35

53

36

54

37# required answerer function

55# required answerer function

searx/engines/duden.pynew file+76

@@ -0,0 +1,76 @@

1+"""

2+ Duden

3+ @website https://www.duden.de

4+ @provide-api no

5+ @using-api no

6+ @results HTML (using search portal)

7+ @stable no (HTML can change)

8+ @parse url, title, content

9+"""

10+

11+from lxml import html, etree

12+import re

13+from searx.engines.xpath import extract_text

14+from searx.url_utils import quote

15+from searx import logger

16+

17+categories = ['general']

18+paging = True

19+language_support = False

20+

21+# search-url

22+base_url = 'https://www.duden.de/'

23+search_url = base_url + 'suchen/dudenonline/{query}?page={offset}'

24+

25+

26+def request(query, params):

27+ '''pre-request callback

28+ params<dict>:

29+ method : POST/GET

30+ headers : {}

31+ data : {} # if method == POST

32+ url : ''

33+ category: 'search category'

34+ pageno : 1 # number of the requested page

35+ '''

36+

37+ offset = (params['pageno'] - 1)

38+ params['url'] = search_url.format(offset=offset, query=quote(query))

39+ return params

40+

41+

42+def response(resp):

43+ '''post-response callback

44+ resp: requests response object

45+ '''

46+ results = []

47+

48+ dom = html.fromstring(resp.text)

49+

50+ try:

51+ number_of_results_string = re.sub('[^0-9]', '', dom.xpath(

52+ '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]

53+ )

54+

55+ results.append({'number_of_results': int(number_of_results_string)})

56+

57+ except:

58+ logger.debug("Couldn't read number of results.")

59+ pass

60+

61+ for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'):

62+ try:

63+ logger.debug("running for %s" % str(result))

64+ link = result.xpath('.//h2/a')[0]

65+ url = link.attrib.get('href')

66+ title = result.xpath('string(.//h2/a)')

67+ content = extract_text(result.xpath('.//p'))

68+ # append result

69+ results.append({'url': url,

70+ 'title': title,

71+ 'content': content})

72+ except:

73+ logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True))

74+ continue

75+

76+ return results

searx/settings.yml+5

@@ -714,6 +714,11 @@ engines:

714 shortcut : 1337x

715 disabled : True

716

717+ - name : Duden

718+ engine : duden

719+ shortcut : du

720+ disabled : True

721+

717# - name : yacy

722# - name : yacy

718# engine : yacy

723# engine : yacy

719# shortcut : ya

724# shortcut : ya

tests/unit/engines/test_duden.pynew file+41

@@ -0,0 +1,41 @@

1+from collections import defaultdict

2+import mock

3+from searx.engines import duden

4+from searx.testing import SearxTestCase

5+from datetime import datetime

6+

7+

8+class TestDudenEngine(SearxTestCase):

9+

10+ def test_request(self):

11+ query = 'Haus'

12+ dic = defaultdict(dict)

13+ dic['pageno'] = 1

14+ params = duden.request(query, dic)

15+ self.assertTrue('url' in params)

16+ self.assertTrue(query in params['url'])

17+ self.assertTrue('duden.de' in params['url'])

18+

19+ def test_response(self):

20+ resp = mock.Mock(text='<html></html>')

21+ self.assertEqual(duden.response(resp), [])

22+

23+ html = """

24+ <section class="wide">

25+ <h2><a href="https://this.is.the.url/" class="hidden-link"><strong>This is the title</strong> also here</a></h2>

26+ <p>This is the <strong>content</strong></p>

27+ <a href="https://this.is.the.url/">Zum vollständigen Artikel</a>

28+ </section>

29+ """

30+

31+ resp = mock.Mock(text=html)

32+ results = duden.response(resp)

33+

34+ self.assertEqual(len(results), 1)

35+ self.assertEqual(type(results), list)

36+

37+ # testing result (dictionary entry)

38+ r = results[0]

39+ self.assertEqual(r['url'], 'https://this.is.the.url/')

40+ self.assertEqual(r['title'], 'This is the title also here')

41+ self.assertEqual(r['content'], 'This is the content')