git.iko.soy/searx
on master help sign in
Merge pull request #1260 from MarcAbonce/engine-fixes

[fix] Engine fixes

commite5def5b0191a36315d300457cdfdf68aebbc9a4c authorAdam Tauber <asciimoo@gmail.com> dateMon 9 Apr 2018, 09:34 (9 years ago) parent283f6c99687786
6 files changed+13−17
compare with master
unifiedsplit
searx/engines/google_news.py+2−2 searx/engines/wikidata.py+6−7 searx/engines/xpath.py+1−1 searx/settings.yml+1 tests/unit/engines/test_google_news.py+2−2 tests/unit/engines/test_wikidata.py+1−5
searx/engines/google_news.py+2−2
@@ -68,8 +68,8 @@ def response(resp):
6868 for result in dom.xpath('//div[@class="g"]|//div[@class="g _cy"]'):
6969 try:
7070 r = {
71− 'url': result.xpath('.//a[@class="l _PMs"]')[0].attrib.get("href"),
72− 'title': ''.join(result.xpath('.//a[@class="l _PMs"]//text()')),
71+ 'url': result.xpath('.//a[@class="l lLrAF"]')[0].attrib.get("href"),
72+ 'title': ''.join(result.xpath('.//a[@class="l lLrAF"]//text()')),
7373 'content': ''.join(result.xpath('.//div[@class="st"]//text()')),
7474 }
7575 except:
searx/engines/wikidata.py+6−7
@@ -27,7 +27,7 @@ result_count = 1
2727# urls
2828wikidata_host = 'https://www.wikidata.org'
2929url_search = wikidata_host \
30− + '/wiki/Special:ItemDisambiguation?{query}'
30+ + '/w/index.php?{query}'
3131
3232wikidata_api = wikidata_host + '/w/api.php'
3333url_detail = wikidata_api\
@@ -40,7 +40,7 @@ url_map = 'https://www.openstreetmap.org/'\
4040url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'
4141
4242# xpaths
43−wikidata_ids_xpath = '//div/ul[@class="wikibase-disambiguation"]/li/a/@title'
43+wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href'
4444title_xpath = '//*[contains(@class,"wikibase-title-label")]'
4545description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'
4646property_xpath = '//div[@id="{propertyid}"]'
@@ -57,22 +57,21 @@ calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
5757
5858
5959def request(query, params):
60− language = match_language(params['language'], supported_languages).split('-')[0]
61−
6260 params['url'] = url_search.format(
63− query=urlencode({'label': query, 'language': language}))
61+ query=urlencode({'search': query}))
6462 return params
6563
6664
6765def response(resp):
6866 results = []
6967 html = fromstring(resp.text)
70− wikidata_ids = html.xpath(wikidata_ids_xpath)
68+ search_results = html.xpath(wikidata_ids_xpath)
7169
7270 language = match_language(resp.search_params['language'], supported_languages).split('-')[0]
7371
7472 # TODO: make requests asynchronous to avoid timeout when result_count > 1
75− for wikidata_id in wikidata_ids[:result_count]:
73+ for search_result in search_results[:result_count]:
74+ wikidata_id = search_result.split('/')[-1]
7675 url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))
7776 htmlresponse = get(url)
7877 jsonresponse = loads(htmlresponse.text)
searx/engines/xpath.py+1−1
@@ -53,7 +53,7 @@ def extract_url(xpath_results, search_url):
5353 if url.startswith('//'):
5454 # add http or https to this kind of url //example.com/
5555 parsed_search_url = urlparse(search_url)
56− url = u'{0}:{1}'.format(parsed_search_url.scheme, url)
56+ url = u'{0}:{1}'.format(parsed_search_url.scheme or 'http', url)
5757 elif url.startswith('/'):
5858 # fix relative url to the search engine
5959 url = urljoin(search_url, url)
searx/settings.yml+1
@@ -174,6 +174,7 @@ engines:
174174 - name : wikidata
175175 engine : wikidata
176176 shortcut : wd
177+ timeout : 3.0
177178 weight : 2
178179
179180 - name : duckduckgo
tests/unit/engines/test_google_news.py+2−2
@@ -42,7 +42,7 @@ class TestGoogleNewsEngine(SearxTestCase):
4242 <div class="ts _JGs _JHs _tJs _KGs _jHs">
4343 <div class="_hJs">
4444 <h3 class="r _gJs">
45− <a class="l _PMs" href="https://example.com/" onmousedown="return rwt(this,'','','','11','AFQjCNEyehpzD5cJK1KUfXBx9RmsbqqG9g','','0ahUKEwjB58OR54HWAhWnKJoKHSQhAMY4ChCpAggiKAAwAA','','',event)">Example title</a>
45+ <a class="l lLrAF" href="https://example.com/" onmousedown="return rwt(this,'','','','11','AFQjCNEyehpzD5cJK1KUfXBx9RmsbqqG9g','','0ahUKEwjB58OR54HWAhWnKJoKHSQhAMY4ChCpAggiKAAwAA','','',event)">Example title</a>
4646 </h3>
4747 <div class="slp">
4848 <span class="_OHs _PHs">
@@ -63,7 +63,7 @@ class TestGoogleNewsEngine(SearxTestCase):
6363 </a>
6464 <div class="_hJs">
6565 <h3 class="r _gJs">
66− <a class="l _PMs" href="https://example2.com/" onmousedown="return rwt(this,'','','','12','AFQjCNHObfH7sYmLWI1SC-YhWXKZFRzRjw','','0ahUKEwjB58OR54HWAhWnKJoKHSQhAMY4ChCpAgglKAAwAQ','','',event)">Example title 2</a>
66+ <a class="l lLrAF" href="https://example2.com/" onmousedown="return rwt(this,'','','','12','AFQjCNHObfH7sYmLWI1SC-YhWXKZFRzRjw','','0ahUKEwjB58OR54HWAhWnKJoKHSQhAMY4ChCpAgglKAAwAQ','','',event)">Example title 2</a>
6767 </h3>
6868 <div class="slp">
6969 <span class="_OHs _PHs">
tests/unit/engines/test_wikidata.py+1−5
@@ -9,20 +9,15 @@ from searx.testing import SearxTestCase
99class TestWikidataEngine(SearxTestCase):
1010
1111 def test_request(self):
12− wikidata.supported_languages = ['en', 'es']
1312 query = 'test_query'
1413 dicto = defaultdict(dict)
15− dicto['language'] = 'en-US'
1614 params = wikidata.request(query, dicto)
1715 self.assertIn('url', params)
1816 self.assertIn(query, params['url'])
1917 self.assertIn('wikidata.org', params['url'])
20− self.assertIn('en', params['url'])
2118
22− dicto['language'] = 'es-ES'
2319 params = wikidata.request(query, dicto)
2420 self.assertIn(query, params['url'])
25− self.assertIn('es', params['url'])
2621
2722 # successful cases are not tested here to avoid sending additional requests
2823 def test_response(self):
@@ -31,6 +26,7 @@ class TestWikidataEngine(SearxTestCase):
3126 self.assertRaises(AttributeError, wikidata.response, '')
3227 self.assertRaises(AttributeError, wikidata.response, '[]')
3328
29+ wikidata.supported_languages = ['en', 'es']
3430 response = mock.Mock(text='<html></html>', search_params={"language": "en"})
3531 self.assertEqual(wikidata.response(response), [])
3632