git.iko.soy/searx
on master help sign in
Remove content field from ArchWiki results; reformat code in archlinux.py

Content field in Arch Wiki search results is of no real use, more often than not it contains no usable information and includes too many markup tags which make the text unreadable. It is safe to remove it.

commit8b7dc2acb9f670ba65e4b98eb310f04e4c212bd8 authorKirill Isakov <ukwt@ya.ru> dateFri 25 Mar 2016, 00:38 (11 years ago) parentd748b84
2 files changed+13−21
compare with master
unifiedsplit
searx/engines/archlinux.py+7−10 tests/unit/engines/test_archlinux.py+6−11
searx/engines/archlinux.py+7−10
@@ -3,12 +3,12 @@
33"""
44 Arch Linux Wiki
55
6− @website https://wiki.archlinux.org
7− @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it
8− @using-api no
9− @results HTML
10− @stable no (HTML can change)
11− @parse url, title, content
6+ @website https://wiki.archlinux.org
7+ @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it
8+ @using-api no
9+ @results HTML
10+ @stable no (HTML can change)
11+ @parse url, title
1212"""
1313
1414from urlparse import urljoin
@@ -26,7 +26,6 @@ base_url = 'https://wiki.archlinux.org'
2626# xpath queries
2727xpath_results = '//ul[@class="mw-search-results"]/li'
2828xpath_link = './/div[@class="mw-search-result-heading"]/a'
29−xpath_content = './/div[@class="searchresult"]'
3029
3130
3231# cut 'en' from 'en_US', 'de' from 'de_CH', and so on
@@ -135,10 +134,8 @@ def response(resp):
135134 link = result.xpath(xpath_link)[0]
136135 href = urljoin(base_url, link.attrib.get('href'))
137136 title = escape(extract_text(link))
138− content = escape(extract_text(result.xpath(xpath_content)))
139137
140138 results.append({'url': href,
141− 'title': title,
142− 'content': content})
139+ 'title': title})
143140
144141 return results
tests/unit/engines/test_archlinux.py+6−11
@@ -18,7 +18,7 @@ class TestArchLinuxEngine(SearxTestCase):
1818 def test_request(self):
1919 query = 'test_query'
2020 dic = defaultdict(dict)
21− dic['pageno'] = 0
21+ dic['pageno'] = 1
2222 dic['language'] = 'en_US'
2323 params = archlinux.request(query, dic)
2424 self.assertTrue('url' in params)
@@ -31,10 +31,8 @@ class TestArchLinuxEngine(SearxTestCase):
3131 self.assertTrue(domain in params['url'])
3232
3333 def test_response(self):
34− response = mock.Mock(text='<html></html>')
35− response.search_params = {
36− 'language': 'en_US'
37− }
34+ response = mock.Mock(text='<html></html>',
35+ search_params={'language': 'en_US'})
3836 self.assertEqual(archlinux.response(response), [])
3937
4038 html = """
@@ -79,18 +77,15 @@ class TestArchLinuxEngine(SearxTestCase):
7977 expected = [
8078 {
8179 'title': 'ATI',
82− 'url': 'https://wiki.archlinux.org/index.php/ATI',
83− 'content': 'Lorem ipsum dolor sit amet'
80+ 'url': 'https://wiki.archlinux.org/index.php/ATI'
8481 },
8582 {
8683 'title': 'Frequently asked questions',
87− 'url': 'https://wiki.archlinux.org/index.php/Frequently_asked_questions',
88− 'content': 'CPUs with AMDs instruction set "AMD64"'
84+ 'url': 'https://wiki.archlinux.org/index.php/Frequently_asked_questions'
8985 },
9086 {
9187 'title': 'CPU frequency scaling',
92− 'url': 'https://wiki.archlinux.org/index.php/CPU_frequency_scaling',
93− 'content': 'ondemand for AMD and older Intel CPU'
88+ 'url': 'https://wiki.archlinux.org/index.php/CPU_frequency_scaling'
9489 }
9590 ]
9691