git.iko.soy/searx
on master help sign in
Add Arch Linux Wiki search engine
d748b84Kirill Isakov
AUTHORS.rst+1
compare with master
unifiedsplit
@@ -41,3 +41,4 @@ generally made searx better:
41- @GreenLunar
41- @GreenLunar
42- Noemi Vanyi
42- Noemi Vanyi
43- Kang-min Liu
43- Kang-min Liu
44+- Kirill Isakov
searx/engines/archlinux.pynew file+144
@@ -0,0 +1,144 @@
1+# -*- coding: utf-8 -*-
2+
3+"""
4+ Arch Linux Wiki
5+
6+ @website https://wiki.archlinux.org
7+ @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it
8+ @using-api no
9+ @results HTML
10+ @stable no (HTML can change)
11+ @parse url, title, content
12+"""
13+
14+from urlparse import urljoin
15+from cgi import escape
16+from urllib import urlencode
17+from lxml import html
18+from searx.engines.xpath import extract_text
19+
20+# engine dependent config
21+categories = ['it']
22+language_support = True
23+paging = True
24+base_url = 'https://wiki.archlinux.org'
25+
26+# xpath queries
27+xpath_results = '//ul[@class="mw-search-results"]/li'
28+xpath_link = './/div[@class="mw-search-result-heading"]/a'
29+xpath_content = './/div[@class="searchresult"]'
30+
31+
32+# cut 'en' from 'en_US', 'de' from 'de_CH', and so on
33+def locale_to_lang_code(locale):
34+ if locale.find('_') >= 0:
35+ locale = locale.split('_')[0]
36+ return locale
37+
38+# wikis for some languages were moved off from the main site, we need to make
39+# requests to correct URLs to be able to get results in those languages
40+lang_urls = {
41+ 'all': {
42+ 'base': 'https://wiki.archlinux.org',
43+ 'search': '/index.php?title=Special:Search&offset={offset}&{query}'
44+ },
45+ 'de': {
46+ 'base': 'https://wiki.archlinux.de',
47+ 'search': '/index.php?title=Spezial:Suche&offset={offset}&{query}'
48+ },
49+ 'fr': {
50+ 'base': 'https://wiki.archlinux.fr',
51+ 'search': '/index.php?title=Spécial:Recherche&offset={offset}&{query}'
52+ },
53+ 'ja': {
54+ 'base': 'https://wiki.archlinuxjp.org',
55+ 'search': '/index.php?title=特別:検索&offset={offset}&{query}'
56+ },
57+ 'ro': {
58+ 'base': 'http://wiki.archlinux.ro',
59+ 'search': '/index.php?title=Special:Căutare&offset={offset}&{query}'
60+ },
61+ 'tr': {
62+ 'base': 'http://archtr.org/wiki',
63+ 'search': '/index.php?title=Özel:Ara&offset={offset}&{query}'
64+ }
65+}
66+
67+
68+# get base & search URLs for selected language
69+def get_lang_urls(language):
70+ if language in lang_urls:
71+ return lang_urls[language]
72+ return lang_urls['all']
73+
74+# Language names to build search requests for
75+# those languages which are hosted on the main site.
76+main_langs = {
77+ 'ar': 'العربية',
78+ 'bg': 'Български',
79+ 'cs': 'Česky',
80+ 'da': 'Dansk',
81+ 'el': 'Ελληνικά',
82+ 'es': 'Español',
83+ 'he': 'עברית',
84+ 'hr': 'Hrvatski',
85+ 'hu': 'Magyar',
86+ 'it': 'Italiano',
87+ 'ko': '한국어',
88+ 'lt': 'Lietuviškai',
89+ 'nl': 'Nederlands',
90+ 'pl': 'Polski',
91+ 'pt': 'Português',
92+ 'ru': 'Русский',
93+ 'sl': 'Slovenský',
94+ 'th': 'ไทย',
95+ 'uk': 'Українська',
96+ 'zh': '简体中文'
97+}
98+
99+
100+# do search-request
101+def request(query, params):
102+ # translate the locale (e.g. 'en_US') to language code ('en')
103+ language = locale_to_lang_code(params['language'])
104+
105+ # if our language is hosted on the main site, we need to add its name
106+ # to the query in order to narrow the results to that language
107+ if language in main_langs:
108+ query += '(' + main_langs[language] + ')'
109+
110+ # prepare the request parameters
111+ query = urlencode({'search': query})
112+ offset = (params['pageno'] - 1) * 20
113+
114+ # get request URLs for our language of choice
115+ urls = get_lang_urls(language)
116+ search_url = urls['base'] + urls['search']
117+
118+ params['url'] = search_url.format(query=query, offset=offset)
119+
120+ return params
121+
122+
123+# get response from search-request
124+def response(resp):
125+ # get the base URL for the language in which request was made
126+ language = locale_to_lang_code(resp.search_params['language'])
127+ base_url = get_lang_urls(language)['base']
128+
129+ results = []
130+
131+ dom = html.fromstring(resp.text)
132+
133+ # parse results
134+ for result in dom.xpath(xpath_results):
135+ link = result.xpath(xpath_link)[0]
136+ href = urljoin(base_url, link.attrib.get('href'))
137+ title = escape(extract_text(link))
138+ content = escape(extract_text(result.xpath(xpath_content)))
139+
140+ results.append({'url': href,
141+ 'title': title,
142+ 'content': content})
143+
144+ return results
searx/settings.yml+4
@@ -34,6 +34,10 @@ outgoing: # communication with search engines
34# - 1.1.1.2
34# - 1.1.1.2
35
35
36engines:
36engines:
37+ - name : arch linux wiki
38+ engine : archlinux
39+ shortcut : al
40+
37 - name : wikipedia
41 - name : wikipedia
38 engine : mediawiki
42 engine : mediawiki
39 shortcut : wp
43 shortcut : wp
tests/unit/engines/test_archlinux.pynew file+111
@@ -0,0 +1,111 @@
1+from collections import defaultdict
2+import mock
3+from searx.engines import archlinux
4+from searx.testing import SearxTestCase
5+
6+domains = {
7+ 'all': 'https://wiki.archlinux.org',
8+ 'de': 'https://wiki.archlinux.de',
9+ 'fr': 'https://wiki.archlinux.fr',
10+ 'ja': 'https://wiki.archlinuxjp.org',
11+ 'ro': 'http://wiki.archlinux.ro',
12+ 'tr': 'http://archtr.org/wiki'
13+}
14+
15+
16+class TestArchLinuxEngine(SearxTestCase):
17+
18+ def test_request(self):
19+ query = 'test_query'
20+ dic = defaultdict(dict)
21+ dic['pageno'] = 0
22+ dic['language'] = 'en_US'
23+ params = archlinux.request(query, dic)
24+ self.assertTrue('url' in params)
25+ self.assertTrue(query in params['url'])
26+ self.assertTrue('wiki.archlinux.org' in params['url'])
27+
28+ for lang, domain in domains.iteritems():
29+ dic['language'] = lang
30+ params = archlinux.request(query, dic)
31+ self.assertTrue(domain in params['url'])
32+
33+ def test_response(self):
34+ response = mock.Mock(text='<html></html>')
35+ response.search_params = {
36+ 'language': 'en_US'
37+ }
38+ self.assertEqual(archlinux.response(response), [])
39+
40+ html = """
41+ <ul class="mw-search-results">
42+ <li>
43+ <div class="mw-search-result-heading">
44+ <a href="/index.php/ATI" title="ATI">ATI</a>
45+ </div>
46+ <div class="searchresult">
47+ Lorem ipsum dolor sit amet
48+ </div>
49+ <div class="mw-search-result-data">
50+ 30 KB (4,630 words) - 19:04, 17 March 2016</div>
51+ </li>
52+ <li>
53+ <div class="mw-search-result-heading">
54+ <a href="/index.php/Frequently_asked_questions" title="Frequently asked questions">
55+ Frequently asked questions
56+ </a>
57+ </div>
58+ <div class="searchresult">
59+ CPUs with AMDs instruction set "AMD64"
60+ </div>
61+ <div class="mw-search-result-data">
62+ 17 KB (2,722 words) - 20:13, 21 March 2016
63+ </div>
64+ </li>
65+ <li>
66+ <div class="mw-search-result-heading">
67+ <a href="/index.php/CPU_frequency_scaling" title="CPU frequency scaling">CPU frequency scaling</a>
68+ </div>
69+ <div class="searchresult">
70+ ondemand for AMD and older Intel CPU
71+ </div>
72+ <div class="mw-search-result-data">
73+ 15 KB (2,319 words) - 23:46, 16 March 2016
74+ </div>
75+ </li>
76+ </ul>
77+ """
78+
79+ expected = [
80+ {
81+ 'title': 'ATI',
82+ 'url': 'https://wiki.archlinux.org/index.php/ATI',
83+ 'content': 'Lorem ipsum dolor sit amet'
84+ },
85+ {
86+ 'title': 'Frequently asked questions',
87+ 'url': 'https://wiki.archlinux.org/index.php/Frequently_asked_questions',
88+ 'content': 'CPUs with AMDs instruction set "AMD64"'
89+ },
90+ {
91+ 'title': 'CPU frequency scaling',
92+ 'url': 'https://wiki.archlinux.org/index.php/CPU_frequency_scaling',
93+ 'content': 'ondemand for AMD and older Intel CPU'
94+ }
95+ ]
96+
97+ response = mock.Mock(text=html)
98+ response.search_params = {
99+ 'language': 'en_US'
100+ }
101+ results = archlinux.response(response)
102+
103+ self.assertEqual(type(results), list)
104+ self.assertEqual(len(results), len(expected))
105+
106+ i = 0
107+ for exp in expected:
108+ res = results[i]
109+ i += 1
110+ for key, value in exp.iteritems():
111+ self.assertEqual(res[key], value)