12f891d · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #1669 from dalf/engine-fixes

Engine fixes

commit12f891da8496fad1f1fa24eeb52b1b19f03f8678 authorAlexandre Flament <alex@al-f.net> dateMon 5 Aug 2019, 15:57 (7 years ago) parent91a59ff 1cee2c1

7 files changed+58−50

compare with master

unifiedsplit

searx/engines/arxiv.py+1−1 searx/engines/bing.py−2 searx/engines/dictzone.py+1−1 searx/engines/fdroid.py+14−15 searx/settings.yml+8−8 tests/unit/engines/test_arxiv.py+1−1 tests/unit/engines/test_fdroid.py+33−22

searx/engines/arxiv.py+1−1

@@ -29,7 +29,7 @@ def request(query, params):

2929 # basic search

3030 offset = (params['pageno'] - 1) * number_of_results

3131

32− string_args = dict(query=query,

32+ string_args = dict(query=query.decode('utf-8'),

3333 offset=offset,

3434 number_of_results=number_of_results)

3535

searx/engines/bing.py−2

@@ -47,8 +47,6 @@ def request(query, params):

4747

4848 params['url'] = base_url + search_path

4949

50− params['headers']['User-Agent'] = gen_useragent('Windows NT 6.3; WOW64')

51−

5250 return params

5351

5452

searx/engines/dictzone.py+1−1

@@ -15,7 +15,7 @@ from searx.utils import is_valid_lang

1515from searx.url_utils import urljoin

1616

1717categories = ['general']

18−url = u'http://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'

18+url = u'https://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'

1919weight = 100

2020

2121parser_re = re.compile(b'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)

searx/engines/fdroid.py+14−15

@@ -18,13 +18,13 @@ categories = ['files']

1818paging = True

1919

2020# search-url

21−base_url = 'https://f-droid.org/'

22−search_url = base_url + 'repository/browse/?{query}'

21+base_url = 'https://search.f-droid.org/'

22+search_url = base_url + '?{query}'

2323

2424

2525# do search-request

2626def request(query, params):

27− query = urlencode({'fdfilter': query, 'fdpage': params['pageno']})

27+ query = urlencode({'q': query, 'page': params['pageno'], 'lang': ''})

2828 params['url'] = search_url.format(query=query)

2929 return params

3030

@@ -35,17 +35,16 @@ def response(resp):

3535

3636 dom = html.fromstring(resp.text)

3737

38− for app in dom.xpath('//div[@id="appheader"]'):

39− url = app.xpath('./ancestor::a/@href')[0]

40− title = app.xpath('./p/span/text()')[0]

41− img_src = app.xpath('.//img/@src')[0]

42−

43− content = extract_text(app.xpath('./p')[0])

44− content = content.replace(title, '', 1).strip()

45−

46− results.append({'url': url,

47− 'title': title,

48− 'content': content,

49− 'img_src': img_src})

38+ for app in dom.xpath('//a[@class="package-header"]'):

39+ app_url = app.xpath('./@href')[0]

40+ app_title = extract_text(app.xpath('./div/h4[@class="package-name"]/text()'))

41+ app_content = extract_text(app.xpath('./div/div/span[@class="package-summary"]')).strip() \

42+ + ' - ' + extract_text(app.xpath('./div/div/span[@class="package-license"]')).strip()

43+ app_img_src = app.xpath('./img[@class="package-icon"]/@src')[0]

44+

45+ results.append({'url': app_url,

46+ 'title': app_title,

47+ 'content': app_content,

48+ 'img_src': app_img_src})

5049

5150 return results

searx/settings.yml+8−8

@@ -204,11 +204,11 @@ engines:

204204 - name : etymonline

205205 engine : xpath

206206 paging : True

207− search_url : http://etymonline.com/?search={query}&p={pageno}

208− url_xpath : //a[contains(@class, "word--")]/@href

209− title_xpath : //p[contains(@class, "word__name--")]/text()

210− content_xpath : //section[contains(@class, "word__defination")]/object

211− first_page_num : 0

207+ search_url : https://etymonline.com/search?page={pageno}&q={query}

208+ url_xpath : //a[contains(@class, "word__name--")]/@href

209+ title_xpath : //a[contains(@class, "word__name--")]

210+ content_xpath : //section[contains(@class, "word__defination")]

211+ first_page_num : 1

212212 shortcut : et

213213 disabled : True

214214

@@ -703,9 +703,9 @@ engines:

703703 shortcut: vo

704704 categories: social media

705705 search_url : https://searchvoat.co/?t={query}

706− url_xpath : //div[@class="entry"]/p/a[@class="title"]/@href

707− title_xpath : //div[@class="entry"]/p/a[@class="title"]

708− content_xpath : //div[@class="entry"]/p/span[@class="domain"]

706+ url_xpath : //div[@class="entry"]/p/a[contains(@class, "title")]/@href

707+ title_xpath : //div[@class="entry"]/p/a[contains(@class, "title")]

708+ content_xpath : //div[@class="entry"]/p/span[@class="domain"]/a/text()

709709 timeout : 10.0

710710 disabled : True

711711

tests/unit/engines/test_arxiv.py+1−1

@@ -8,7 +8,7 @@ from searx.testing import SearxTestCase

88class TestBaseEngine(SearxTestCase):

1010 def test_request(self):

11− query = 'test_query'

11+ query = 'test_query'.encode('utf-8')

1212 dicto = defaultdict(dict)

1313 dicto['pageno'] = 1

1414 params = arxiv.request(query, dicto)

tests/unit/engines/test_fdroid.py+33−22

@@ -13,29 +13,40 @@ class TestFdroidEngine(SearxTestCase):

1313 params = fdroid.request(query, dic)

1414 self.assertTrue('url' in params)

1515 self.assertTrue(query in params['url'])

16− self.assertTrue('f-droid.org' in params['url'])

16+ self.assertTrue('search.f-droid.org' in params['url'])

1717

18− def test_response(self):

18+ def test_response_empty(self):

1919 resp = mock.Mock(text='<html></html>')

2020 self.assertEqual(fdroid.response(resp), [])

2121

22+ def test_response_oneresult(self):

2223 html = """

23− <a href="https://google.com/qwerty">

24− <div id="appheader">

25− <div style="float:left;padding-right:10px;">

26− <img src="http://example.com/image.png"

27− style="width:48px;border:none;">

28− </div>

29− <div style="float:right;">

30− <p>Details...</p>

31− </div>

32− <p style="color:#000000;">

33− <span style="font-size:20px;">Sample title</span>

34− <br>

35− Sample content

36− </p>

37− </div>

38− </a>

24+<!DOCTYPE html>

25+<html>

26+<head>

27+ <title>test</title>

28+</head>

29+<body>

30+ <div class="site-wrapper">

31+ <div class="main-content">

32+ <a class="package-header" href="https://example.com/app.url">

33+ <img class="package-icon" src="https://example.com/appexample.logo.png" />

34+

35+ <div class="package-info">

36+ <h4 class="package-name">

37+ App Example 1

38+ </h4>

39+

40+ <div class="package-desc">

41+ <span class="package-summary">Description App Example 1</span>

42+ <span class="package-license">GPL-3.0-only</span>

43+ </div>

44+ </div>

45+ </a>

46+ </div>

47+ </div>

48+</body>

49+</html>

3950 """

4051

4152 resp = mock.Mock(text=html)

@@ -43,7 +54,7 @@ class TestFdroidEngine(SearxTestCase):

4354

4455 self.assertEqual(type(results), list)

4556 self.assertEqual(len(results), 1)

46− self.assertEqual(results[0]['url'], 'https://google.com/qwerty')

47− self.assertEqual(results[0]['title'], 'Sample title')

48− self.assertEqual(results[0]['content'], 'Sample content')

49− self.assertEqual(results[0]['img_src'], 'http://example.com/image.png')

57+ self.assertEqual(results[0]['url'], 'https://example.com/app.url')

58+ self.assertEqual(results[0]['title'], 'App Example 1')

59+ self.assertEqual(results[0]['content'], 'Description App Example 1 - GPL-3.0-only')

60+ self.assertEqual(results[0]['img_src'], 'https://example.com/appexample.logo.png')