git.iko.soy/searx
on master help sign in
[enh] flickr_noapi: use complete JSON data block, add 'content', 'img_format', 'source', etc. (#1571)
cbc5e13Frank de Lange
searx/engines/flickr_noapi.py+28−23
compare with master
unifiedsplit
@@ -16,8 +16,7 @@ from json import loads
16from time import time
16from time import time
17import re
17import re
18from searx.engines import logger
18from searx.engines import logger
19−from searx.url_utils import urlencode
19+from searx.url_utils import urlencode, unquote
20−
21
20
22logger = logger.getChild('flickr-noapi')
21logger = logger.getChild('flickr-noapi')
23
22
@@ -27,7 +26,7 @@ url = 'https://www.flickr.com/'
27search_url = url + 'search?{query}&page={page}'
26search_url = url + 'search?{query}&page={page}'
28time_range_url = '&min_upload_date={start}&max_upload_date={end}'
27time_range_url = '&min_upload_date={start}&max_upload_date={end}'
29photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
28photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
30−regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
29+modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M)
31image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
30image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
32
31
33paging = True
32paging = True
@@ -57,40 +56,45 @@ def request(query, params):
57def response(resp):
56def response(resp):
58 results = []
57 results = []
59
58
60− matches = regex.search(resp.text)
59+ matches = modelexport_re.search(resp.text)
61
60
62 if matches is None:
61 if matches is None:
63 return results
62 return results
64
63
65 match = matches.group(1)
64 match = matches.group(1)
66− search_results = loads(match)
65+ model_export = loads(match)
67
66
68− if '_data' not in search_results:
67+ if 'legend' not in model_export:
69− return []
68+ return results
70
69
71− photos = search_results['_data']
70+ legend = model_export['legend']
72
71
73− for photo in photos:
72+ # handle empty page
73+ if not legend or not legend[0]:
74+ return results
74
75
75− # In paged configuration, the first pages' photos
76+ for index in legend:
76− # are represented by a None object
77+ photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][int(index[4])]
77− if photo is None:
78+ author = unquote(photo.get('realname', ''))
78− continue
79+ source = unquote(photo.get('username', '')) + ' @ Flickr'
80+ title = unquote(photo.get('title', ''))
81+ content = unquote(photo.get('description', ''))
79
82
80 img_src = None
83 img_src = None
81 # From the biggest to the lowest format
84 # From the biggest to the lowest format
82 for image_size in image_sizes:
85 for image_size in image_sizes:
83 if image_size in photo['sizes']:
86 if image_size in photo['sizes']:
84 img_src = photo['sizes'][image_size]['url']
87 img_src = photo['sizes'][image_size]['url']
88+ img_format = 'jpg ' \
89+ + str(photo['sizes'][image_size]['width']) \
90+ + 'x' \
91+ + str(photo['sizes'][image_size]['height'])
85 break
92 break
86
93
87 if not img_src:
94 if not img_src:
88 logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
95 logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
89 continue
96 continue
90
97
91− if 'ownerNsid' not in photo:
92− continue
93−
94 # For a bigger thumbnail, keep only the url_z, not the url_n
98 # For a bigger thumbnail, keep only the url_z, not the url_n
95 if 'n' in photo['sizes']:
99 if 'n' in photo['sizes']:
96 thumbnail_src = photo['sizes']['n']['url']
100 thumbnail_src = photo['sizes']['n']['url']
@@ -99,19 +103,20 @@ def response(resp):
99 else:
103 else:
100 thumbnail_src = img_src
104 thumbnail_src = img_src
101
105
106+ if 'ownerNsid' not in photo:
107+ # should not happen, disowned photo? Show it anyway
108+ url = img_src
109+ else:
102− url = build_flickr_url(photo['ownerNsid'], photo['id'])
110+ url = build_flickr_url(photo['ownerNsid'], photo['id'])
103−
104− title = photo.get('title', '')
105−
106− author = photo['username']
107
111
108− # append result
109 results.append({'url': url,
112 results.append({'url': url,
110 'title': title,
113 'title': title,
111 'img_src': img_src,
114 'img_src': img_src,
112 'thumbnail_src': thumbnail_src,
115 'thumbnail_src': thumbnail_src,
113− 'content': '',
116+ 'content': content,
114 'author': author,
117 'author': author,
118+ 'source': source,
119+ 'img_format': img_format,
115 'template': 'images.html'})
120 'template': 'images.html'})
116
121
117 return results
122 return results
tests/unit/engines/test_flickr_noapi.py+249−221
@@ -27,116 +27,132 @@ class TestFlickrNoapiEngine(SearxTestCase):
27 self.assertRaises(AttributeError, flickr_noapi.response, '')
27 self.assertRaises(AttributeError, flickr_noapi.response, '')
28 self.assertRaises(AttributeError, flickr_noapi.response, '[]')
28 self.assertRaises(AttributeError, flickr_noapi.response, '[]')
29
29
30− response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":')
30+ response = mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{}}]}}')
31 self.assertEqual(flickr_noapi.response(response), [])
31 self.assertEqual(flickr_noapi.response(response), [])
32
32
33+ response = \
33− response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":')
34+ mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{"_data":[]}}]}}')
34 self.assertEqual(flickr_noapi.response(response), [])
35 self.assertEqual(flickr_noapi.response(response), [])
35
36
36 # everthing is ok test
37 # everthing is ok test
37 json = """
38 json = """
39+ modelExport: {
40+ "legend": [
41+ [
38− "search-photos-lite-models","photos":
42+ "search-photos-lite-models",
39− {
43+ "0",
44+ "photos",
45+ "_data",
46+ "0"
47+ ]
48+ ],
49+ "main": {
50+ "search-photos-lite-models": [
51+ {
52+ "photos": {
40− "_data": [
53+ "_data": [
41− {
54+ {
42− "_flickrModelRegistry": "photo-lite-models",
55+ "_flickrModelRegistry": "photo-lite-models",
43− "title": "This is the title",
56+ "title": "This%20is%20the%20title",
44− "username": "Owner",
57+ "username": "Owner",
45− "pathAlias": "klink692",
58+ "pathAlias": "klink692",
46− "realname": "Owner",
59+ "realname": "Owner",
47− "license": 0,
60+ "license": 0,
48− "ownerNsid": "59729010@N00",
61+ "ownerNsid": "59729010@N00",
49− "canComment": false,
62+ "canComment": false,
50− "commentCount": 14,
63+ "commentCount": 14,
51− "faveCount": 21,
64+ "faveCount": 21,
52− "id": "14001294434",
65+ "id": "14001294434",
53− "sizes": {
66+ "sizes": {
54− "c": {
67+ "c": {
55− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg",
68+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg",
56− "width": 541,
69+ "width": 541,
57− "height": 800,
70+ "height": 800,
58− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg",
71+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg",
59− "key": "c"
72+ "key": "c"
60− },
73+ },
61− "h": {
74+ "h": {
62− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg",
75+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg",
63− "width": 1081,
76+ "width": 1081,
64− "height": 1600,
77+ "height": 1600,
65− "url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg",
78+ "url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg",
66− "key": "h"
79+ "key": "h"
67− },
80+ },
68− "k": {
81+ "k": {
69− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg",
82+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg",
70− "width": 1383,
83+ "width": 1383,
71− "height": 2048,
84+ "height": 2048,
72− "url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg",
85+ "url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg",
73− "key": "k"
86+ "key": "k"
74− },
87+ },
75− "l": {
88+ "l": {
76− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg",
89+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg",
77− "width": 692,
90+ "width": 692,
78− "height": 1024,
91+ "height": 1024,
79− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg",
92+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg",
80− "key": "l"
93+ "key": "l"
81− },
94+ },
82− "m": {
95+ "m": {
83− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg",
96+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg",
84− "width": 338,
97+ "width": 338,
85− "height": 500,
98+ "height": 500,
86− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg",
99+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg",
87− "key": "m"
100+ "key": "m"
88− },
101+ },
89− "n": {
102+ "n": {
90− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg",
103+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg",
91− "width": 216,
104+ "width": 216,
92− "height": 320,
105+ "height": 320,
93− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg",
106+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg",
94− "key": "n"
107+ "key": "n"
95− },
108+ },
96− "q": {
109+ "q": {
97− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg",
110+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg",
98− "width": 150,
111+ "width": 150,
99− "height": 150,
112+ "height": 150,
100− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg",
113+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg",
101− "key": "q"
114+ "key": "q"
102− },
115+ },
103− "s": {
116+ "s": {
104− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg",
117+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg",
105− "width": 162,
118+ "width": 162,
106− "height": 240,
119+ "height": 240,
107− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg",
120+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg",
108− "key": "s"
121+ "key": "s"
109− },
122+ },
110− "sq": {
123+ "sq": {
111− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg",
124+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg",
112− "width": 75,
125+ "width": 75,
113− "height": 75,
126+ "height": 75,
114− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg",
127+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg",
115− "key": "sq"
128+ "key": "sq"
116− },
129+ },
117− "t": {
130+ "t": {
118− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg",
131+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg",
119− "width": 68,
132+ "width": 68,
120− "height": 100,
133+ "height": 100,
121− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg",
134+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg",
122− "key": "t"
135+ "key": "t"
123− },
136+ },
124− "z": {
137+ "z": {
125− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
138+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
126− "width": 433,
139+ "width": 433,
127− "height": 640,
140+ "height": 640,
128− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
141+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
129− "key": "z"
142+ "key": "z"
143+ }
144+ }
145+ }
146+ ]
130 }
147 }
131 }
148 }
132− }
133− ],
149+ ]
134− "fetchedStart": true,
150+ }
135− "fetchedEnd": false,
151+ }
136− "totalItems": "4386039"
137− },"totalItems":
138 """
152 """
139− json = json.replace('\r\n', '').replace('\n', '').replace('\r', '')
153+ # Flickr serves search results in a json block named 'modelExport' buried inside a script tag,
154+ # this json is served as a single line terminating with a comma.
155+ json = ''.join(json.split()) + ',\n'
140 response = mock.Mock(text=json)
156 response = mock.Mock(text=json)
141 results = flickr_noapi.response(response)
157 results = flickr_noapi.response(response)
142 self.assertEqual(type(results), list)
158 self.assertEqual(type(results), list)
@@ -149,37 +165,51 @@ class TestFlickrNoapiEngine(SearxTestCase):
149
165
150 # no n size, only the z size
166 # no n size, only the z size
151 json = """
167 json = """
168+ modelExport: {
169+ "legend": [
170+ [
152− "search-photos-lite-models","photos":
171+ "search-photos-lite-models",
153− {
172+ "0",
173+ "photos",
174+ "_data",
175+ "0"
176+ ]
177+ ],
178+ "main": {
179+ "search-photos-lite-models": [
180+ {
181+ "photos": {
154− "_data": [
182+ "_data": [
155− {
183+ {
156− "_flickrModelRegistry": "photo-lite-models",
184+ "_flickrModelRegistry": "photo-lite-models",
157− "title": "This is the title",
185+ "title": "This%20is%20the%20title",
158− "username": "Owner",
186+ "username": "Owner",
159− "pathAlias": "klink692",
187+ "pathAlias": "klink692",
160− "realname": "Owner",
188+ "realname": "Owner",
161− "license": 0,
189+ "license": 0,
162− "ownerNsid": "59729010@N00",
190+ "ownerNsid": "59729010@N00",
163− "canComment": false,
191+ "canComment": false,
164− "commentCount": 14,
192+ "commentCount": 14,
165− "faveCount": 21,
193+ "faveCount": 21,
166− "id": "14001294434",
194+ "id": "14001294434",
167− "sizes": {
195+ "sizes": {
168− "z": {
196+ "z": {
169− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
197+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
170− "width": 433,
198+ "width": 433,
171− "height": 640,
199+ "height": 640,
172− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
200+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
173− "key": "z"
201+ "key": "z"
202+ }
203+ }
204+ }
205+ ]
174 }
206 }
175 }
207 }
176− }
177− ],
208+ ]
178− "fetchedStart": true,
209+ }
179− "fetchedEnd": false,
210+ }
180− "totalItems": "4386039"
181− },"totalItems":
182 """
211 """
212+ json = ''.join(json.split()) + ',\n'
183 response = mock.Mock(text=json)
213 response = mock.Mock(text=json)
184 results = flickr_noapi.response(response)
214 results = flickr_noapi.response(response)
185 self.assertEqual(type(results), list)
215 self.assertEqual(type(results), list)
@@ -192,37 +222,51 @@ class TestFlickrNoapiEngine(SearxTestCase):
192
222
193 # no z or n size
223 # no z or n size
194 json = """
224 json = """
225+ modelExport: {
226+ "legend": [
227+ [
195− "search-photos-lite-models","photos":
228+ "search-photos-lite-models",
196− {
229+ "0",
230+ "photos",
231+ "_data",
232+ "0"
233+ ]
234+ ],
235+ "main": {
236+ "search-photos-lite-models": [
237+ {
238+ "photos": {
197− "_data": [
239+ "_data": [
198− {
240+ {
199− "_flickrModelRegistry": "photo-lite-models",
241+ "_flickrModelRegistry": "photo-lite-models",
200− "title": "This is the title",
242+ "title": "This%20is%20the%20title",
201− "username": "Owner",
243+ "username": "Owner",
202− "pathAlias": "klink692",
244+ "pathAlias": "klink692",
203− "realname": "Owner",
245+ "realname": "Owner",
204− "license": 0,
246+ "license": 0,
205− "ownerNsid": "59729010@N00",
247+ "ownerNsid": "59729010@N00",
206− "canComment": false,
248+ "canComment": false,
207− "commentCount": 14,
249+ "commentCount": 14,
208− "faveCount": 21,
250+ "faveCount": 21,
209− "id": "14001294434",
251+ "id": "14001294434",
210− "sizes": {
252+ "sizes": {
211− "o": {
253+ "o": {
212− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
254+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
213− "width": 433,
255+ "width": 433,
214− "height": 640,
256+ "height": 640,
215− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
257+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
216− "key": "o"
258+ "key": "o"
259+ }
260+ }
261+ }
262+ ]
217 }
263 }
218 }
264 }
219− }
220− ],
265+ ]
221− "fetchedStart": true,
266+ }
222− "fetchedEnd": false,
267+ }
223− "totalItems": "4386039"
224− },"totalItems":
225 """
268 """
269+ json = ''.join(json.split()) + ',\n'
226 response = mock.Mock(text=json)
270 response = mock.Mock(text=json)
227 results = flickr_noapi.response(response)
271 results = flickr_noapi.response(response)
228 self.assertEqual(type(results), list)
272 self.assertEqual(type(results), list)
@@ -235,30 +279,44 @@ class TestFlickrNoapiEngine(SearxTestCase):
235
279
236 # no image test
280 # no image test
237 json = """
281 json = """
238− "search-photos-lite-models","photos":
282+ modelExport: {
239− {
283+ "legend": [
284+ [
285+ "search-photos-lite-models",
286+ "0",
287+ "photos",
240− "_data": [
288+ "_data",
241− {
289+ "0"
242− "_flickrModelRegistry": "photo-lite-models",
290+ ]
243− "title": "This is the title",
244− "username": "Owner",
245− "pathAlias": "klink692",
246− "realname": "Owner",
247− "license": 0,
248− "ownerNsid": "59729010@N00",
249− "canComment": false,
250− "commentCount": 14,
251− "faveCount": 21,
252− "id": "14001294434",
253− "sizes": {
254− }
255− }
256 ],
291 ],
257− "fetchedStart": true,
292+ "main": {
258− "fetchedEnd": false,
293+ "search-photos-lite-models": [
259− "totalItems": "4386039"
294+ {
260− },"totalItems":
295+ "photos": {
296+ "_data": [
297+ {
298+ "_flickrModelRegistry": "photo-lite-models",
299+ "title": "This is the title",
300+ "username": "Owner",
301+ "pathAlias": "klink692",
302+ "realname": "Owner",
303+ "license": 0,
304+ "ownerNsid": "59729010@N00",
305+ "canComment": false,
306+ "commentCount": 14,
307+ "faveCount": 21,
308+ "id": "14001294434",
309+ "sizes": {
310+ }
311+ }
312+ ]
313+ }
314+ }
315+ ]
316+ }
317+ }
261 """
318 """
319+ json = ''.join(json.split()) + ',\n'
262 response = mock.Mock(text=json)
320 response = mock.Mock(text=json)
263 results = flickr_noapi.response(response)
321 results = flickr_noapi.response(response)
264 self.assertEqual(type(results), list)
322 self.assertEqual(type(results), list)
@@ -266,51 +324,20 @@ class TestFlickrNoapiEngine(SearxTestCase):
266
324
267 # null test
325 # null test
268 json = """
326 json = """
269− "search-photos-models","photos":
327+ modelExport: {
270− {
271− "_data": [null],
328+ "legend": [null],
272− "fetchedStart": true,
329+ "main": {
273− "fetchedEnd": false,
330+ "search-photos-lite-models": [
274− "totalItems": "4386039"
331+ {
275− },"totalItems":
332+ "photos": {
276− """
333+ "_data": [null]
277− response = mock.Mock(text=json)
278− results = flickr_noapi.response(response)
279− self.assertEqual(type(results), list)
280− self.assertEqual(len(results), 0)
281−
282− # no ownerNsid test
283− json = """
284− "search-photos-lite-models","photos":
285− {
286− "_data": [
287− {
288− "_flickrModelRegistry": "photo-lite-models",
289− "title": "This is the title",
290− "username": "Owner",
291− "pathAlias": "klink692",
292− "realname": "Owner",
293− "license": 0,
294− "canComment": false,
295− "commentCount": 14,
296− "faveCount": 21,
297− "id": "14001294434",
298− "sizes": {
299− "o": {
300− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
301− "width": 433,
302− "height": 640,
303− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
304− "key": "o"
305 }
334 }
306 }
335 }
307− }
308− ],
336+ ]
309− "fetchedStart": true,
337+ }
310− "fetchedEnd": false,
338+ }
311− "totalItems": "4386039"
312− },"totalItems":
313 """
339 """
340+ json = ''.join(json.split()) + ',\n'
314 response = mock.Mock(text=json)
341 response = mock.Mock(text=json)
315 results = flickr_noapi.response(response)
342 results = flickr_noapi.response(response)
316 self.assertEqual(type(results), list)
343 self.assertEqual(type(results), list)
@@ -323,6 +350,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
323 "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}
350 "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}
324 ]}
351 ]}
325 """
352 """
353+ json = ''.join(json.split()) + ',\n'
326 response = mock.Mock(text=json)
354 response = mock.Mock(text=json)
327 results = flickr_noapi.response(response)
355 results = flickr_noapi.response(response)
328 self.assertEqual(type(results), list)
356 self.assertEqual(type(results), list)