git.iko.soy/searx
on master help sign in
Merge pull request #219 from pointhi/new_engines

New engines: gigablast and blekko_images

commit5f801d7ea0ee1e4b4bec7f52c45ca3f601fc5cdc authorAdam Tauber <asciimoo@gmail.com> dateMon 9 Feb 2015, 10:46 (12 years ago) parent7c075aadd4686a
6 files changed+252
compare with master
unifiedsplit
searx/engines/blekko_images.py+56 searx/engines/gigablast.py+63 searx/settings.yml+9 searx/tests/engines/test_blekko_images.py+65 searx/tests/engines/test_gigablast.py+57 searx/tests/test_engines.py+2
searx/engines/blekko_images.pynew file+56
@@ -0,0 +1,56 @@
1+## Blekko (Images)
2+#
3+# @website https://blekko.com
4+# @provide-api yes (inofficial)
5+#
6+# @using-api yes
7+# @results JSON
8+# @stable yes
9+# @parse url, title, img_src
10+
11+from json import loads
12+from urllib import urlencode
13+
14+# engine dependent config
15+categories = ['images']
16+paging = True
17+
18+# search-url
19+base_url = 'https://blekko.com'
20+search_url = '/api/images?{query}&c={c}'
21+
22+
23+# do search-request
24+def request(query, params):
25+ c = (params['pageno'] - 1) * 48
26+
27+ params['url'] = base_url +\
28+ search_url.format(query=urlencode({'q': query}),
29+ c=c)
30+
31+ if params['pageno'] != 1:
32+ params['url'] += '&page={pageno}'.format(pageno=(params['pageno']-1))
33+
34+ return params
35+
36+
37+# get response from search-request
38+def response(resp):
39+ results = []
40+
41+ search_results = loads(resp.text)
42+
43+ # return empty array if there are no results
44+ if not search_results:
45+ return []
46+
47+ for result in search_results:
48+ # append result
49+ results.append({'url': result['page_url'],
50+ 'title': result['title'],
51+ 'content': '',
52+ 'img_src': result['url'],
53+ 'template': 'images.html'})
54+
55+ # return results
56+ return results
searx/engines/gigablast.pynew file+63
@@ -0,0 +1,63 @@
1+## Gigablast (Web)
2+#
3+# @website http://gigablast.com
4+# @provide-api yes (http://gigablast.com/api.html)
5+#
6+# @using-api yes
7+# @results XML
8+# @stable yes
9+# @parse url, title, content
10+
11+from urllib import urlencode
12+from cgi import escape
13+from lxml import etree
14+
15+# engine dependent config
16+categories = ['general']
17+paging = True
18+number_of_results = 5
19+
20+# search-url
21+base_url = 'http://gigablast.com/'
22+search_string = 'search?{query}&n={number_of_results}&s={offset}&xml=1&qh=0'
23+
24+# specific xpath variables
25+results_xpath = '//response//result'
26+url_xpath = './/url'
27+title_xpath = './/title'
28+content_xpath = './/sum'
29+
30+
31+# do search-request
32+def request(query, params):
33+ offset = (params['pageno'] - 1) * number_of_results
34+
35+ search_path = search_string.format(
36+ query=urlencode({'q': query}),
37+ offset=offset,
38+ number_of_results=number_of_results)
39+
40+ params['url'] = base_url + search_path
41+
42+ return params
43+
44+
45+# get response from search-request
46+def response(resp):
47+ results = []
48+
49+ dom = etree.fromstring(resp.content)
50+
51+ # parse results
52+ for result in dom.xpath(results_xpath):
53+ url = result.xpath(url_xpath)[0].text
54+ title = result.xpath(title_xpath)[0].text
55+ content = escape(result.xpath(content_xpath)[0].text)
56+
57+ # append result
58+ results.append({'url': url,
59+ 'title': title,
60+ 'content': content})
61+
62+ # return results
63+ return results
searx/settings.yml+9
@@ -33,6 +33,11 @@ engines:
3333 locale : en-US
3434 shortcut : bin
3535
36+ - name : blekko images
37+ engine : blekko_images
38+ locale : en-US
39+ shortcut : bli
40+
3641 - name : btdigg
3742 engine : btdigg
3843 shortcut : bt
@@ -103,6 +108,10 @@ engines:
103108 shortcut : gf
104109 disabled : True
105110
111+ - name : gigablast
112+ engine : gigablast
113+ shortcut : gb
114+
106115 - name : github
107116 engine : github
108117 shortcut : gh
searx/tests/engines/test_blekko_images.pynew file+65
@@ -0,0 +1,65 @@
1+from collections import defaultdict
2+import mock
3+from searx.engines import blekko_images
4+from searx.testing import SearxTestCase
5+
6+
7+class TestBlekkoImagesEngine(SearxTestCase):
8+
9+ def test_request(self):
10+ query = 'test_query'
11+ dicto = defaultdict(dict)
12+ dicto['pageno'] = 0
13+ params = blekko_images.request(query, dicto)
14+ self.assertTrue('url' in params)
15+ self.assertTrue(query in params['url'])
16+ self.assertTrue('blekko.com' in params['url'])
17+
18+ def test_response(self):
19+ self.assertRaises(AttributeError, blekko_images.response, None)
20+ self.assertRaises(AttributeError, blekko_images.response, [])
21+ self.assertRaises(AttributeError, blekko_images.response, '')
22+ self.assertRaises(AttributeError, blekko_images.response, '[]')
23+
24+ response = mock.Mock(text='[]')
25+ self.assertEqual(blekko_images.response(response), [])
26+
27+ json = """
28+ [
29+ {
30+ "c": 1,
31+ "page_url": "http://result_url.html",
32+ "title": "Photo title",
33+ "tn_url": "http://ts1.mm.bing.net/th?id=HN.608050619474382748&pid=15.1",
34+ "url": "http://result_image.jpg"
35+ },
36+ {
37+ "c": 2,
38+ "page_url": "http://companyorange.simpsite.nl/OSM",
39+ "title": "OSM",
40+ "tn_url": "http://ts2.mm.bing.net/th?id=HN.608048068264919461&pid=15.1",
41+ "url": "http://simpsite.nl/userdata2/58985/Home/OSM.bmp"
42+ },
43+ {
44+ "c": 3,
45+ "page_url": "http://invincible.webklik.nl/page/osm",
46+ "title": "OSM",
47+ "tn_url": "http://ts1.mm.bing.net/th?id=HN.608024514657649476&pid=15.1",
48+ "url": "http://www.webklik.nl/user_files/2009_09/65324/osm.gif"
49+ },
50+ {
51+ "c": 4,
52+ "page_url": "http://www.offshorenorway.no/event/companyDetail/id/12492",
53+ "title": "Go to OSM Offshore AS homepage",
54+ "tn_url": "http://ts2.mm.bing.net/th?id=HN.608054265899847285&pid=15.1",
55+ "url": "http://www.offshorenorway.no/firmalogo/OSM-logo.png"
56+ }
57+ ]
58+ """
59+ response = mock.Mock(text=json)
60+ results = blekko_images.response(response)
61+ self.assertEqual(type(results), list)
62+ self.assertEqual(len(results), 4)
63+ self.assertEqual(results[0]['title'], 'Photo title')
64+ self.assertEqual(results[0]['url'], 'http://result_url.html')
65+ self.assertEqual(results[0]['img_src'], 'http://result_image.jpg')
searx/tests/engines/test_gigablast.pynew file+57
@@ -0,0 +1,57 @@
1+from collections import defaultdict
2+import mock
3+from searx.engines import gigablast
4+from searx.testing import SearxTestCase
5+
6+
7+class TestGigablastEngine(SearxTestCase):
8+
9+ def test_request(self):
10+ query = 'test_query'
11+ dicto = defaultdict(dict)
12+ dicto['pageno'] = 0
13+ params = gigablast.request(query, dicto)
14+ self.assertTrue('url' in params)
15+ self.assertTrue(query in params['url'])
16+ self.assertTrue('gigablast.com' in params['url'])
17+
18+ def test_response(self):
19+ self.assertRaises(AttributeError, gigablast.response, None)
20+ self.assertRaises(AttributeError, gigablast.response, [])
21+ self.assertRaises(AttributeError, gigablast.response, '')
22+ self.assertRaises(AttributeError, gigablast.response, '[]')
23+
24+ response = mock.Mock(content='<response></response>')
25+ self.assertEqual(gigablast.response(response), [])
26+
27+ response = mock.Mock(content='<response></response>')
28+ self.assertEqual(gigablast.response(response), [])
29+
30+ xml = """<?xml version="1.0" encoding="UTF-8" ?>
31+ <response>
32+ <hits>5941888</hits>
33+ <moreResultsFollow>1</moreResultsFollow>
34+ <result>
35+ <title><![CDATA[This should be the title]]></title>
36+ <sum><![CDATA[This should be the content.]]></sum>
37+ <url><![CDATA[http://this.should.be.the.link/]]></url>
38+ <size>90.5</size>
39+ <docId>145414002633</docId>
40+ <siteId>2660021087</siteId>
41+ <domainId>2660021087</domainId>
42+ <spidered>1320519373</spidered>
43+ <indexed>1320519373</indexed>
44+ <pubdate>4294967295</pubdate>
45+ <isModDate>0</isModDate>
46+ <language><![CDATA[English]]></language>
47+ <charset><![CDATA[UTF-8]]></charset>
48+ </result>
49+ </response>
50+ """
51+ response = mock.Mock(content=xml)
52+ results = gigablast.response(response)
53+ self.assertEqual(type(results), list)
54+ self.assertEqual(len(results), 1)
55+ self.assertEqual(results[0]['title'], 'This should be the title')
56+ self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
57+ self.assertEqual(results[0]['content'], 'This should be the content.')
searx/tests/test_engines.py+2
@@ -1,6 +1,7 @@
11from searx.tests.engines.test_bing import * # noqa
22from searx.tests.engines.test_bing_images import * # noqa
33from searx.tests.engines.test_bing_news import * # noqa
4+from searx.tests.engines.test_blekko_images import * # noqa
45from searx.tests.engines.test_btdigg import * # noqa
56from searx.tests.engines.test_dailymotion import * # noqa
67from searx.tests.engines.test_deezer import * # noqa
@@ -9,6 +10,7 @@ from searx.tests.engines.test_digg import * # noqa
910from searx.tests.engines.test_dummy import * # noqa
1011from searx.tests.engines.test_flickr import * # noqa
1112from searx.tests.engines.test_flickr_noapi import * # noqa
13+from searx.tests.engines.test_gigablast import * # noqa
1214from searx.tests.engines.test_github import * # noqa
1315from searx.tests.engines.test_www1x import * # noqa
1416from searx.tests.engines.test_google_images import * # noqa