git.iko.soy/searx
on master help sign in
Merge branch '500px_rewrite' of github.com:asciimoo/searx
commite12ea9a5106a8b58080c0395c2fcf3f2a84808fe authorAdam Tauber <asciimoo@gmail.com> dateSun 11 Dec 2016, 03:31 (10 years ago) parentf6e9c079b2bd68
2 files changed+32−73
compare with master
unifiedsplit
searx/engines/www500px.py+25−17 tests/unit/engines/test_www500px.py+7−56
searx/engines/www500px.py+25−17
@@ -12,12 +12,9 @@
1212 @todo rewrite to api
1313"""
1414
15−
15+from json import loads
1616from urllib import urlencode
1717from urlparse import urljoin
18−from lxml import html
19−import re
20−from searx.engines.xpath import extract_text
2118
2219# engine dependent config
2320categories = ['images']
@@ -25,13 +22,27 @@ paging = True
2522
2623# search-url
2724base_url = 'https://500px.com'
28−search_url = base_url + '/search?search?page={pageno}&type=photos&{query}'
25+search_url = 'https://api.500px.com/v1/photos/search?type=photos'\
26+ '&{query}'\
27+ '&image_size%5B%5D=4'\
28+ '&image_size%5B%5D=20'\
29+ '&image_size%5B%5D=21'\
30+ '&image_size%5B%5D=1080'\
31+ '&image_size%5B%5D=1600'\
32+ '&image_size%5B%5D=2048'\
33+ '&include_states=true'\
34+ '&formats=jpeg%2Clytro'\
35+ '&include_tags=true'\
36+ '&exclude_nude=true'\
37+ '&page={pageno}'\
38+ '&rpp=50'\
39+ '&sdk_key=b68e60cff4c929bedea36ca978830c5caca790c3'
2940
3041
3142# do search-request
3243def request(query, params):
3344 params['url'] = search_url.format(pageno=params['pageno'],
34− query=urlencode({'q': query}))
45+ query=urlencode({'term': query}))
3546
3647 return params
3748
@@ -40,19 +51,16 @@ def request(query, params):
4051def response(resp):
4152 results = []
4253
43− dom = html.fromstring(resp.text)
44− regex = re.compile(r'3\.jpg.*$')
54+ response_json = loads(resp.text)
4555
4656 # parse results
47− for result in dom.xpath('//div[@class="photo"]'):
48− link = result.xpath('.//a')[0]
49− url = urljoin(base_url, link.attrib.get('href'))
50− title = extract_text(result.xpath('.//div[@class="title"]'))
51− thumbnail_src = link.xpath('.//img')[0].attrib.get('src')
52− # To have a bigger thumbnail, uncomment the next line
53− # thumbnail_src = regex.sub('4.jpg', thumbnail_src)
54− content = extract_text(result.xpath('.//div[@class="info"]'))
55− img_src = regex.sub('2048.jpg', thumbnail_src)
57+ for result in response_json['photos']:
58+ url = urljoin(base_url, result['url'])
59+ title = result['name']
60+ # last index is the biggest resolution
61+ img_src = result['image_url'][-1]
62+ thumbnail_src = result['image_url'][0]
63+ content = result['description'] or ''
5664
5765 # append result
5866 results.append({'url': url,
tests/unit/engines/test_www500px.py+7−56
@@ -22,62 +22,13 @@ class TestWww500pxImagesEngine(SearxTestCase):
2222 self.assertRaises(AttributeError, www500px.response, '')
2323 self.assertRaises(AttributeError, www500px.response, '[]')
2424
25− response = mock.Mock(text='<html></html>')
26− self.assertEqual(www500px.response(response), [])
27−
28− html = """
29− <div class="photo">
30− <a href="/this.should.be.the.url" data-ga-category="Photo Thumbnail" data-ga-action="Title">
31− <img src="https://image.url/3.jpg?v=0" />
32− </a>
33− <div class="details">
34− <div class="inside">
35− <div class="title">
36− <a href="/photo/64312705/branch-out-by-oliver-turpin?feature=">
37− This is the title
38− </a>
39− </div>
40− <div class="info">
41− <a href="/ChronicleUK" data-ga-action="Image" data-ga-category="Photo Thumbnail">
42− This is the content
43− </a>
44− </div>
45− <div class="rating">44.8</div>
46− </div>
47− </div>
48− </div>
49− """
50− response = mock.Mock(text=html)
25+ json = """
26+{"current_page":1,"total_pages":1000,"total_items":862178,"photos":[{"id":64531569,"user_id":111147,"name":"Grand Canyon Afternoon","description":"Looking west on a very windy winter afternoon.","camera":"Canon EOS 5D Mark II","lens":"EF24-105mm f/4L IS USM","focal_length":"28","iso":"200","shutter_speed":"1/1250","aperture":"6.3","times_viewed":4809,"rating":48.5,"status":1,"created_at":"2014-03-22T03:44:46-04:00","category":8,"location":null,"latitude":36.0323916666667,"longitude":-111.85273,"taken_at":"2014-02-27T14:10:43-05:00","hi_res_uploaded":2,"for_sale":true,"width":5476,"height":3651,"votes_count":108,"favorites_count":35,"comments_count":5,"nsfw":false,"sales_count":0,"for_sale_date":null,"highest_rating":91.9,"highest_rating_date":"2014-03-22T22:34:54-04:00","license_type":0,"converted":31,"collections_count":10,"crop_version":0,"privacy":false,"profile":true,"image_url":["https://drscdn.500px.org/photo/64531569/w%3D70_h%3D70/449d50817f28d85395e23bbb415b3cdb?v=0","https://drscdn.500px.org/photo/64531569/q%3D50_w%3D140_h%3D140/3e3e123734a596644ede78105268bdb2?v=0","https://drscdn.500px.org/photo/64531569/q%3D80_h%3D300/2ce2f61714aebdca710967dfdc3efb04","https://drscdn.500px.org/photo/64531569/q%3D80_h%3D450/c8ec030441f2c68b9bd40a114903348a","https://drscdn.500px.org/photo/64531569/q%3D80_h%3D600/ab6562d0581b359679ecc8ef2e939396","https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1000/bd7dbc54a505e041a8c9a70dfa434272","https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1500/eb4d7f8f6a32d3e5c168c2cb55d29c12","https://drscdn.500px.org/photo/64531569/q%3D80_m%3D2000/d519f91b8a568e7357a8a7fa1aabbe74","https://drscdn.500px.org/photo/64531569/m%3D2048/4c52fb18cc2b2b6f91a0d04609786507","https://drscdn.500px.org/photo/64531569/m%3D900/fb620ae39569ab4a421e9170a94b1a0f","https://drscdn.500px.org/photo/64531569/m%3D900_s%3D1_k%3D1_a%3D1/02b95ce64db090c1f94f890960974612?v=0"],"images":[{"size":1,"url":"https://drscdn.500px.org/photo/64531569/w%3D70_h%3D70/449d50817f28d85395e23bbb415b3cdb?v=0","https_url":"https://drscdn.500px.org/photo/64531569/w%3D70_h%3D70/449d50817f28d85395e23bbb415b3cdb?v=0","format":"jpeg"},{"size":2,"url":"https://drscdn.500px.org/photo/64531569/q%3D50_w%3D140_h%3D140/3e3e123734a596644ede78105268bdb2?v=0","https_url":"https://drscdn.500px.org/photo/64531569/q%3D50_w%3D140_h%3D140/3e3e123734a596644ede78105268bdb2?v=0","format":"jpeg"},{"size":4,"url":"https://drscdn.500px.org/photo/64531569/m%3D900/fb620ae39569ab4a421e9170a94b1a0f","https_url":"https://drscdn.500px.org/photo/64531569/m%3D900/fb620ae39569ab4a421e9170a94b1a0f","format":"jpeg"},{"size":14,"url":"https://drscdn.500px.org/photo/64531569/m%3D900_s%3D1_k%3D1_a%3D1/02b95ce64db090c1f94f890960974612?v=0","https_url":"https://drscdn.500px.org/photo/64531569/m%3D900_s%3D1_k%3D1_a%3D1/02b95ce64db090c1f94f890960974612?v=0","format":"jpeg"},{"size":31,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D450/c8ec030441f2c68b9bd40a114903348a","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D450/c8ec030441f2c68b9bd40a114903348a","format":"jpeg"},{"size":32,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D300/2ce2f61714aebdca710967dfdc3efb04","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D300/2ce2f61714aebdca710967dfdc3efb04","format":"jpeg"},{"size":33,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D600/ab6562d0581b359679ecc8ef2e939396","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_h%3D600/ab6562d0581b359679ecc8ef2e939396","format":"jpeg"},{"size":34,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1000/bd7dbc54a505e041a8c9a70dfa434272","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1000/bd7dbc54a505e041a8c9a70dfa434272","format":"jpeg"},{"size":35,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1500/eb4d7f8f6a32d3e5c168c2cb55d29c12","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D1500/eb4d7f8f6a32d3e5c168c2cb55d29c12","format":"jpeg"},{"size":36,"url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D2000/d519f91b8a568e7357a8a7fa1aabbe74","https_url":"https://drscdn.500px.org/photo/64531569/q%3D80_m%3D2000/d519f91b8a568e7357a8a7fa1aabbe74","format":"jpeg"},{"size":2048,"url":"https://drscdn.500px.org/photo/64531569/m%3D2048/4c52fb18cc2b2b6f91a0d04609786507","https_url":"https://drscdn.500px.org/photo/64531569/m%3D2048/4c52fb18cc2b2b6f91a0d04609786507","format":"jpeg"}],"url":"/photo/64531569/grand-canyon-afternoon-by-todd-hakala","positive_votes_count":108,"converted_bits":31,"tags":["landscape","river","arizona","canyon","grand","colorado","south","southwest","az","west","rim","CanonGetaway"],"watermark":false,"image_format":"jpeg","licensing_requested":false,"licensing_suggested":false,"is_free_photo":false,"user":{"id":111147,"username":"ToddHakala","firstname":"Todd","lastname":"Hakala","city":"Albuquerque","country":"US","usertype":0,"fullname":"Todd Hakala","userpic_url":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/1.jpg?2","userpic_https_url":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/1.jpg?2","cover_url":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/cover_2048.jpg?8","upgrade_status":1,"store_on":true,"affection":5217,"avatars":{"default":{"https":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/1.jpg?2"},"large":{"https":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/2.jpg?2"},"small":{"https":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/3.jpg?2"},"tiny":{"https":"https://pacdn.500px.org/111147/ea167926a64ce9b32e44cbec61e3af4f75b762cb/4.jpg?2"}},"followers_count":171}}]}
27+ """ # noqa
28+ response = mock.Mock(text=json)
5129 results = www500px.response(response)
5230 self.assertEqual(type(results), list)
5331 self.assertEqual(len(results), 1)
54− self.assertEqual(results[0]['title'], 'This is the title')
55− self.assertEqual(results[0]['url'], 'https://500px.com/this.should.be.the.url')
56− self.assertEqual(results[0]['content'], 'This is the content')
57− self.assertEqual(results[0]['thumbnail_src'], 'https://image.url/3.jpg?v=0')
58− self.assertEqual(results[0]['img_src'], 'https://image.url/2048.jpg')
59−
60− html = """
61− <a href="/this.should.be.the.url" data-ga-category="Photo Thumbnail" data-ga-action="Title">
62− <img src="https://image.url/3.jpg?v=0" />
63− </a>
64− <div class="details">
65− <div class="inside">
66− <div class="title">
67− <a href="/photo/64312705/branch-out-by-oliver-turpin?feature=">
68− This is the title
69− </a>
70− </div>
71− <div class="info">
72− <a href="/ChronicleUK" data-ga-action="Image" data-ga-category="Photo Thumbnail">
73− Oliver Turpin
74− </a>
75− </div>
76− <div class="rating">44.8</div>
77− </div>
78− </div>
79− """
80− response = mock.Mock(text=html)
81− results = www500px.response(response)
82− self.assertEqual(type(results), list)
83− self.assertEqual(len(results), 0)
32+ self.assertEqual(results[0]['title'], u'Grand Canyon Afternoon')
33+ self.assertEqual(results[0]['url'], 'https://500px.com/photo/64531569/grand-canyon-afternoon-by-todd-hakala')
34+ self.assertEqual(results[0]['content'], u'Looking west on a very windy winter afternoon.')