git.iko.soy/searx
on master help sign in
[enh] add gigablast engine
commit04f7118d0a0693906ef57fa83f01d29eb366a45e authorThomas Pointhuber <thomas.pointhuber@gmx.at> dateSun 8 Feb 2015, 14:12 (12 years ago) parent7c075aa
4 files changed+125
compare with master
unifiedsplit
searx/engines/gigablast.py+63 searx/settings.yml+4 searx/tests/engines/test_gigablast.py+57 searx/tests/test_engines.py+1
searx/engines/gigablast.pynew file+63
@@ -0,0 +1,63 @@
1+## Gigablast (Web)
2+#
3+# @website http://gigablast.com
4+# @provide-api yes (http://gigablast.com/api.html)
5+#
6+# @using-api yes
7+# @results XML
8+# @stable yes
9+# @parse url, title, content
10+
11+from urllib import urlencode
12+from cgi import escape
13+from lxml import etree
14+
15+# engine dependent config
16+categories = ['general']
17+paging = True
18+number_of_results = 5
19+
20+# search-url
21+base_url = 'http://gigablast.com/'
22+search_string = 'search?{query}&n={number_of_results}&s={offset}&xml=1&qh=0'
23+
24+# specific xpath variables
25+results_xpath = '//response//result'
26+url_xpath = './/url'
27+title_xpath = './/title'
28+content_xpath = './/sum'
29+
30+
31+# do search-request
32+def request(query, params):
33+ offset = (params['pageno'] - 1) * number_of_results
34+
35+ search_path = search_string.format(
36+ query=urlencode({'q': query}),
37+ offset=offset,
38+ number_of_results=number_of_results)
39+
40+ params['url'] = base_url + search_path
41+
42+ return params
43+
44+
45+# get response from search-request
46+def response(resp):
47+ results = []
48+
49+ dom = etree.fromstring(resp.content)
50+
51+ # parse results
52+ for result in dom.xpath(results_xpath):
53+ url = result.xpath(url_xpath)[0].text
54+ title = result.xpath(title_xpath)[0].text
55+ content = escape(result.xpath(content_xpath)[0].text)
56+
57+ # append result
58+ results.append({'url': url,
59+ 'title': title,
60+ 'content': content})
61+
62+ # return results
63+ return results
searx/settings.yml+4
@@ -103,6 +103,10 @@ engines:
103103 shortcut : gf
104104 disabled : True
105105
106+ - name : gigablast
107+ engine : gigablast
108+ shortcut : gb
109+
106110 - name : github
107111 engine : github
108112 shortcut : gh
searx/tests/engines/test_gigablast.pynew file+57
@@ -0,0 +1,57 @@
1+from collections import defaultdict
2+import mock
3+from searx.engines import gigablast
4+from searx.testing import SearxTestCase
5+
6+
7+class TestGigablastEngine(SearxTestCase):
8+
9+ def test_request(self):
10+ query = 'test_query'
11+ dicto = defaultdict(dict)
12+ dicto['pageno'] = 0
13+ params = gigablast.request(query, dicto)
14+ self.assertTrue('url' in params)
15+ self.assertTrue(query in params['url'])
16+ self.assertTrue('gigablast.com' in params['url'])
17+
18+ def test_response(self):
19+ self.assertRaises(AttributeError, gigablast.response, None)
20+ self.assertRaises(AttributeError, gigablast.response, [])
21+ self.assertRaises(AttributeError, gigablast.response, '')
22+ self.assertRaises(AttributeError, gigablast.response, '[]')
23+
24+ response = mock.Mock(content='<response></response>')
25+ self.assertEqual(gigablast.response(response), [])
26+
27+ response = mock.Mock(content='<response></response>')
28+ self.assertEqual(gigablast.response(response), [])
29+
30+ xml = """<?xml version="1.0" encoding="UTF-8" ?>
31+ <response>
32+ <hits>5941888</hits>
33+ <moreResultsFollow>1</moreResultsFollow>
34+ <result>
35+ <title><![CDATA[This should be the title]]></title>
36+ <sum><![CDATA[This should be the content.]]></sum>
37+ <url><![CDATA[http://this.should.be.the.link/]]></url>
38+ <size>90.5</size>
39+ <docId>145414002633</docId>
40+ <siteId>2660021087</siteId>
41+ <domainId>2660021087</domainId>
42+ <spidered>1320519373</spidered>
43+ <indexed>1320519373</indexed>
44+ <pubdate>4294967295</pubdate>
45+ <isModDate>0</isModDate>
46+ <language><![CDATA[English]]></language>
47+ <charset><![CDATA[UTF-8]]></charset>
48+ </result>
49+ </response>
50+ """
51+ response = mock.Mock(content=xml)
52+ results = gigablast.response(response)
53+ self.assertEqual(type(results), list)
54+ self.assertEqual(len(results), 1)
55+ self.assertEqual(results[0]['title'], 'This should be the title')
56+ self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
57+ self.assertEqual(results[0]['content'], 'This should be the content.')
searx/tests/test_engines.py+1
@@ -9,6 +9,7 @@ from searx.tests.engines.test_digg import * # noqa
99from searx.tests.engines.test_dummy import * # noqa
1010from searx.tests.engines.test_flickr import * # noqa
1111from searx.tests.engines.test_flickr_noapi import * # noqa
12+from searx.tests.engines.test_gigablast import * # noqa
1213from searx.tests.engines.test_github import * # noqa
1314from searx.tests.engines.test_www1x import * # noqa
1415from searx.tests.engines.test_google_images import * # noqa