git.iko.soy/searx
on master help sign in
Merge pull request #531 from guyou/add-doku-engine

Add doku engine

commit5544fdb75610bb66d05392289e0f0ad48c13ccf6 authorAdam Tauber <asciimoo@gmail.com> dateTue 12 Apr 2016, 13:54 (11 years ago) parent11c226251cb832
4 files changed+171
compare with master
unifiedsplit
AUTHORS.rst+1 searx/engines/doku.py+84 searx/settings.yml+7 tests/unit/engines/test_doku.py+79
AUTHORS.rst+1
@@ -42,3 +42,4 @@ generally made searx better:
4242- Noemi Vanyi
4343- Kang-min Liu
4444- Kirill Isakov
45+- Guilhem Bonnefille
searx/engines/doku.pynew file+84
@@ -0,0 +1,84 @@
1+# Doku Wiki
2+#
3+# @website https://www.dokuwiki.org/
4+# @provide-api yes
5+# (https://www.dokuwiki.org/devel:xmlrpc)
6+#
7+# @using-api no
8+# @results HTML
9+# @stable yes
10+# @parse (general) url, title, content
11+
12+from urllib import urlencode
13+from lxml.html import fromstring
14+from searx.engines.xpath import extract_text
15+
16+# engine dependent config
17+categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'
18+paging = False
19+language_support = False
20+number_of_results = 5
21+
22+# search-url
23+# Doku is OpenSearch compatible
24+base_url = 'http://localhost:8090'
25+search_url = '/?do=search'\
26+ '&{query}'
27+# TODO '&startRecord={offset}'\
28+# TODO '&maximumRecords={limit}'\
29+
30+
31+# do search-request
32+def request(query, params):
33+
34+ params['url'] = base_url +\
35+ search_url.format(query=urlencode({'id': query}))
36+
37+ return params
38+
39+
40+# get response from search-request
41+def response(resp):
42+ results = []
43+
44+ doc = fromstring(resp.text)
45+
46+ # parse results
47+ # Quickhits
48+ for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'):
49+ try:
50+ res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]
51+ except:
52+ continue
53+
54+ if not res_url:
55+ continue
56+
57+ title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))
58+
59+ # append result
60+ results.append({'title': title,
61+ 'content': "",
62+ 'url': base_url + res_url})
63+
64+ # Search results
65+ for r in doc.xpath('//dl[@class="search_results"]/*'):
66+ try:
67+ if r.tag == "dt":
68+ res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]
69+ title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))
70+ elif r.tag == "dd":
71+ content = extract_text(r.xpath('.'))
72+
73+ # append result
74+ results.append({'title': title,
75+ 'content': content,
76+ 'url': base_url + res_url})
77+ except:
78+ continue
79+
80+ if not res_url:
81+ continue
82+
83+ # return results
84+ return results
searx/settings.yml+7
@@ -337,6 +337,13 @@ engines:
337337# number_of_results : 5
338338# timeout : 3.0
339339
340+# Doku engine lets you access to any Doku wiki instance:
341+# A public one or a privete/corporate one.
342+# - name : ubuntuwiki
343+# engine : doku
344+# shortcut : uw
345+# base_url : 'http://doc.ubuntu-fr.org'
346+
340347locales:
341348 en : English
342349 bg : Български (Bulgarian)
tests/unit/engines/test_doku.pynew file+79
@@ -0,0 +1,79 @@
1+# -*- coding: utf-8 -*-
2+from collections import defaultdict
3+import mock
4+from searx.engines import doku
5+from searx.testing import SearxTestCase
6+
7+
8+class TestDokuEngine(SearxTestCase):
9+
10+ def test_request(self):
11+ query = 'test_query'
12+ dicto = defaultdict(dict)
13+ params = doku.request(query, dicto)
14+ self.assertIn('url', params)
15+ self.assertIn(query, params['url'])
16+
17+ def test_response(self):
18+ self.assertRaises(AttributeError, doku.response, None)
19+ self.assertRaises(AttributeError, doku.response, [])
20+ self.assertRaises(AttributeError, doku.response, '')
21+ self.assertRaises(AttributeError, doku.response, '[]')
22+
23+ response = mock.Mock(text='<html></html>')
24+ self.assertEqual(doku.response(response), [])
25+
26+ html = u"""
27+ <div class="search_quickresult">
28+ <h3>Pages trouvées :</h3>
29+ <ul class="search_quickhits">
30+ <li> <a href="/xfconf-query" class="wikilink1" title="xfconf-query">xfconf-query</a></li>
31+ </ul>
32+ <div class="clearer"></div>
33+ </div>
34+ """
35+ response = mock.Mock(text=html)
36+ results = doku.response(response)
37+ expected = [{'content': '', 'title': 'xfconf-query', 'url': 'http://localhost:8090/xfconf-query'}]
38+ self.assertEqual(doku.response(response), expected)
39+
40+ html = u"""
41+ <dl class="search_results">
42+ <dt><a href="/xvnc?s[]=query" class="wikilink1" title="xvnc">xvnc</a>: 40 Occurrences trouvées</dt>
43+ <dd>er = /usr/bin/Xvnc
44+ server_args = -inetd -<strong class="search_hit">query</strong> localhost -geometry 640x480 ... er = /usr/bin/Xvnc
45+ server_args = -inetd -<strong class="search_hit">query</strong> localhost -geometry 800x600 ... er = /usr/bin/Xvnc
46+ server_args = -inetd -<strong class="search_hit">query</strong> localhost -geometry 1024x768 ... er = /usr/bin/Xvnc
47+ server_args = -inetd -<strong class="search_hit">query</strong> localhost -geometry 1280x1024 -depth 8 -Sec</dd>
48+ <dt><a href="/postfix_mysql_tls_sasl_1404?s[]=query"
49+ class="wikilink1"
50+ title="postfix_mysql_tls_sasl_1404">postfix_mysql_tls_sasl_1404</a>: 14 Occurrences trouvées</dt>
51+ <dd>tdepasse
52+ hosts = 127.0.0.1
53+ dbname = postfix
54+ <strong class="search_hit">query</strong> = SELECT goto FROM alias WHERE address='%s' AND a... tdepasse
55+ hosts = 127.0.0.1
56+ dbname = postfix
57+ <strong class="search_hit">query</strong> = SELECT domain FROM domain WHERE domain='%s'
58+ #optional <strong class="search_hit">query</strong> to use when relaying for backup MX
59+ #<strong class="search_hit">query</strong> = SELECT domain FROM domain WHERE domain='%s' and backupmx =</dd>
60+ <dt><a href="/bind9?s[]=query" class="wikilink1" title="bind9">bind9</a>: 12 Occurrences trouvées</dt>
61+ <dd> printcmd
62+;; Got answer:
63+;; -&gt;&gt;HEADER&lt;&lt;- opcode: <strong class="search_hit">QUERY</strong>, status: NOERROR, id: 13427
64+;; flags: qr aa rd ra; <strong class="search_hit">QUERY</strong>: 1, ANSWER: 1, AUTHORITY: 1, ADDITIONAL: 1
65+
66+[...]
67+
68+;; <strong class="search_hit">Query</strong> time: 1 msec
69+;; SERVER: 127.0.0.1#53(127.0.0.1)
70+;... par la requête (<strong class="search_hit">Query</strong> time) , entre la première et la deuxième requête.</dd>
71+ </dl>
72+ """
73+ response = mock.Mock(text=html)
74+ results = doku.response(response)
75+ self.assertEqual(type(results), list)
76+ self.assertEqual(len(results), 3)
77+ self.assertEqual(results[0]['title'], 'xvnc')
78+# FIXME self.assertEqual(results[0]['url'], u'http://this.should.be.the.link/ű')
79+# FIXME self.assertEqual(results[0]['content'], 'This should be the content.')