git.iko.soy/searx
on master help sign in
[fix] replace the dead google images ajax api with a working one
439cf05Adam Tauber
searx/engines/google_images.py+29−30
compare with master
unifiedsplit
@@ -2,41 +2,42 @@
2 Google (Images)
2 Google (Images)
3
3
4 @website https://www.google.com
4 @website https://www.google.com
5− @provide-api yes (https://developers.google.com/web-search/docs/),
5+ @provide-api yes (https://developers.google.com/custom-search/)
6− deprecated!
7
6
8 @using-api yes
7 @using-api yes
9− @results JSON
8+ @results HTML chunk
10− @stable yes (but deprecated)
9+ @stable no
11 @parse url, title, img_src
10 @parse url, title, img_src
12"""
11"""
13
12
14−from urllib import urlencode, unquote
13+from urllib import urlencode
14+from urlparse import parse_qs
15from json import loads
15from json import loads
16+from lxml import html
16
17
17# engine dependent config
18# engine dependent config
18categories = ['images']
19categories = ['images']
19paging = True
20paging = True
20safesearch = True
21safesearch = True
21
22
22−# search-url
23+search_url = 'https://www.google.com/search'\
23−url = 'https://ajax.googleapis.com/'
24+ '?{query}'\
24−search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe={safesearch}&filter=off&{query}'
25+ '&tbm=isch'\
26+ '&ijn=1'\
27+ '&start={offset}'
25
28
26
29
27# do search-request
30# do search-request
28def request(query, params):
31def request(query, params):
29− offset = (params['pageno'] - 1) * 8
32+ offset = (params['pageno'] - 1) * 100
30−
31− if params['safesearch'] == 0:
32− safesearch = 'off'
33− else:
34− safesearch = 'on'
35
33
36 params['url'] = search_url.format(query=urlencode({'q': query}),
34 params['url'] = search_url.format(query=urlencode({'q': query}),
37 offset=offset,
35 offset=offset,
38 safesearch=safesearch)
36 safesearch=safesearch)
39
37
38+ if safesearch and params['safesearch']:
39+ params['url'] += '&' + urlencode({'safe': 'active'})
40+
40 return params
41 return params
41
42
42
43
@@ -44,30 +45,28 @@ def request(query, params):
44def response(resp):
45def response(resp):
45 results = []
46 results = []
46
47
47− search_res = loads(resp.text)
48+ dom = html.fromstring(resp.text)
48−
49− # return empty array if there are no results
50− if not search_res.get('responseData', {}).get('results'):
51− return []
52
49
53 # parse results
50 # parse results
54− for result in search_res['responseData']['results']:
51+ for result in dom.xpath('//div[@data-ved]'):
55− href = result['originalContextUrl']
52+ data_url = result.xpath('./a/@href')[0]
56− title = result['title']
53+ data_query = {k: v[0] for k, v in parse_qs(data_url.split('?', 1)[1]).iteritems()}
57− if 'url' not in result:
54+
58− continue
55+ metadata = loads(result.xpath('./div[@class="rg_meta"]/text()')[0])
56+
59− thumbnail_src = result['tbUrl']
57+ thumbnail_src = metadata['tu']
60
58
61 # http to https
59 # http to https
62 thumbnail_src = thumbnail_src.replace("http://", "https://")
60 thumbnail_src = thumbnail_src.replace("http://", "https://")
63
61
64 # append result
62 # append result
65− results.append({'url': href,
63+ results.append({'url': data_query['imgrefurl'],
66− 'title': title,
64+ 'title': metadata['pt'],
67− 'content': result['content'],
65+ 'content': metadata['s'],
68− 'thumbnail_src': thumbnail_src,
66+ 'thumbnail_src': metadata['tu'],
69− 'img_src': unquote(result['url']),
67+ 'img_src': data_query['imgurl'],
70 'template': 'images.html'})
68 'template': 'images.html'})
71
69
72 # return results
70 # return results
71+ print len(results)
73 return results
72 return results
searx/tests/engines/test_google_images.py+28−83
@@ -10,15 +10,15 @@ class TestGoogleImagesEngine(SearxTestCase):
10 query = 'test_query'
10 query = 'test_query'
11 dicto = defaultdict(dict)
11 dicto = defaultdict(dict)
12 dicto['pageno'] = 1
12 dicto['pageno'] = 1
13+ dicto['safesearch'] = 1
13 params = google_images.request(query, dicto)
14 params = google_images.request(query, dicto)
14 self.assertIn('url', params)
15 self.assertIn('url', params)
15 self.assertIn(query, params['url'])
16 self.assertIn(query, params['url'])
16− self.assertIn('googleapis.com', params['url'])
17− self.assertIn('safe=on', params['url'])
17+ self.assertIn('safe=active', params['url'])
18
18
19 dicto['safesearch'] = 0
19 dicto['safesearch'] = 0
20 params = google_images.request(query, dicto)
20 params = google_images.request(query, dicto)
21− self.assertIn('safe=off', params['url'])
21+ self.assertNotIn('safe', params['url'])
22
22
23 def test_response(self):
23 def test_response(self):
24 self.assertRaises(AttributeError, google_images.response, None)
24 self.assertRaises(AttributeError, google_images.response, None)
@@ -26,88 +26,33 @@ class TestGoogleImagesEngine(SearxTestCase):
26 self.assertRaises(AttributeError, google_images.response, '')
26 self.assertRaises(AttributeError, google_images.response, '')
27 self.assertRaises(AttributeError, google_images.response, '[]')
27 self.assertRaises(AttributeError, google_images.response, '[]')
28
28
29− response = mock.Mock(text='{}')
29+ response = mock.Mock(text='<div></div>')
30 self.assertEqual(google_images.response(response), [])
30 self.assertEqual(google_images.response(response), [])
31
31
32− response = mock.Mock(text='{"data": []}')
32+ html = """
33− self.assertEqual(google_images.response(response), [])
33+<div style="display:none">
34−
34+ <div eid="fWhnVq4Shqpp3pWo4AM" id="isr_scm_1" style="display:none"></div>
35− json = """
35+ <div data-cei="fWhnVq4Shqpp3pWo4AM" class="rg_add_chunk"><!--m-->
36− {
36+ <div class="rg_di rg_el ivg-i" data-ved="0ahUKEwjuxPWQts3JAhUGVRoKHd4KCjwQMwgDKAAwAA">
37− "responseData": {
37+ <a href="/imgres?imgurl=http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png&amp;imgrefurl=http://www.clker.com/clipart-south-arrow.html&amp;h=598&amp;w=504&amp;tbnid=bQWQ9wz9loJmjM:&amp;docid=vlONkeBtERfDuM&amp;ei=fWhnVq4Shqpp3pWo4AM&amp;tbm=isch" jsaction="fire.ivg_o;mouseover:str.hmov;mouseout:str.hmou" class="rg_l"><img data-src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRsxy3gKnEX0lrwwpRxdPWyLJ8iZ--PXZ-ThbBA2_xXDG_bdQutMQ" data-sz="f" name="bQWQ9wz9loJmjM:" class="rg_i" alt="Image result for south" jsaction="load:str.tbn" onload="google.aft&&google.aft(this)">
38− "results": [
38+ <div class="_aOd rg_ilm">
39− {
39+ <div class="rg_ilmbg"><span class="rg_ilmn"> 504&nbsp;&#215;&nbsp;598 - clker.com </span>
40− "GsearchResultClass": "GimageSearch",
40+ </div>
41− "width": "400",
41+ </div>
42− "height": "400",
42+ </a>
43− "imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ",
43+ <div class="rg_meta">
44− "tbWidth": "124",
44+ {"id":"bQWQ9wz9loJmjM:","isu":"clker.com","ity":"png","md":"/search?tbs\u003dsbi:AMhZZit7u1mHyop9pQisu-5idR-8W_1Itvwc3afChmsjQYPx_1yYMzBvUZgtkcGoojqekKZ-6n_1rjX9ySH0OWA_1eO5OijFY6BBDw_1GApr6xxb1bXJcBcj-DiguMoXWW7cZSG7MRQbwnI5SoDZNXcv_1xGszy886I7NVb_1oRKSliTHtzqbXAxhvYreM","msu":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAQSEgltBZD3DP2WgiG-U42R4G0RFw","oh":598,"os":"13KB","ow":504,"pt":"South Arrow Clip Art at Clker.com - vector clip art online ...","rid":"vlONkeBtERfDuM","s":"Download this image as:","sc":1,"si":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAESEgltBZD3DP2WgiG-U42R4G0RFw","th":245,"tu":"https://thumbnail.url/","tw":206}
45− "tbHeight": "124",
45+ </div>
46− "unescapedUrl": "http://unescaped.url.jpg",
46+ </div><!--n--><!--m-->
47− "url": "http://image.url.jpg",
47+ </div>
48− "visibleUrl": "insolitebuzz.fr",
48+</div>
49− "title": "This is the title",
50− "titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire",
51− "originalContextUrl": "http://this.is.the.url",
52− "content": "<b>test</b>",
53− "contentNoFormatting": "test",
54− "tbUrl": "http://thumbnail.url"
55− }
56− ]
57− },
58− "responseDetails": null,
59− "responseStatus": 200
60− }
61− """
49+ """ # noqa
62− response = mock.Mock(text=json)
50+ response = mock.Mock(text=html)
63 results = google_images.response(response)
51 results = google_images.response(response)
64 self.assertEqual(type(results), list)
52 self.assertEqual(type(results), list)
65 self.assertEqual(len(results), 1)
53 self.assertEqual(len(results), 1)
66− self.assertEqual(results[0]['title'], 'This is the title')
54+ self.assertEqual(results[0]['title'], u'South Arrow Clip Art at Clker.com - vector clip art online ...')
67− self.assertEqual(results[0]['url'], 'http://this.is.the.url')
55+ self.assertEqual(results[0]['url'], 'http://www.clker.com/clipart-south-arrow.html')
68− self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url')
56+ self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url/')
69− self.assertEqual(results[0]['img_src'], 'http://image.url.jpg')
57+ self.assertEqual(results[0]['img_src'], 'http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png')
70− self.assertEqual(results[0]['content'], '<b>test</b>')
58+ self.assertEqual(results[0]['content'], 'Download this image as:')
71−
72− json = """
73− {
74− "responseData": {
75− "results": [
76− {
77− "GsearchResultClass": "GimageSearch",
78− "width": "400",
79− "height": "400",
80− "imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ",
81− "tbWidth": "124",
82− "tbHeight": "124",
83− "unescapedUrl": "http://unescaped.url.jpg",
84− "visibleUrl": "insolitebuzz.fr",
85− "title": "This is the title",
86− "titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire",
87− "originalContextUrl": "http://this.is.the.url",
88− "content": "<b>test</b>",
89− "contentNoFormatting": "test",
90− "tbUrl": "http://thumbnail.url"
91− }
92− ]
93− },
94− "responseDetails": null,
95− "responseStatus": 200
96− }
97− """
98− response = mock.Mock(text=json)
99− results = google_images.response(response)
100− self.assertEqual(type(results), list)
101− self.assertEqual(len(results), 0)
102−
103− json = """
104− {
105− "responseData": {},
106− "responseDetails": null,
107− "responseStatus": 200
108− }
109− """
110− response = mock.Mock(text=json)
111− results = google_images.response(response)
112− self.assertEqual(type(results), list)
113− self.assertEqual(len(results), 0)