git.iko.soy/searx
on master help sign in
[enh] flickr_noapi: use complete JSON data block, add 'content', 'img_format', 'source', etc. (#1571)

Fetch complete JSON data block, use legend to extract images. Unquote urlencoded strings. Add image description as 'content'. Add 'img_format' and 'source' data (needs PR #1567 to enable this data to be displayed). Show images which lack ownerid instead of discarding them.

commitcbc5e13275fe3ea5bbab2c7665432398d2179c24 authorFrank de Lange <github-f@unternet.org> dateSun 28 Jul 2019, 10:42 (7 years ago) parentc6ac39b
2 files changed+277−244
compare with master
unifiedsplit
searx/engines/flickr_noapi.py+28−23 tests/unit/engines/test_flickr_noapi.py+249−221
searx/engines/flickr_noapi.py+28−23
@@ -16,8 +16,7 @@ from json import loads
1616from time import time
1717import re
1818from searx.engines import logger
19−from searx.url_utils import urlencode
20−
19+from searx.url_utils import urlencode, unquote
2120
2221logger = logger.getChild('flickr-noapi')
2322
@@ -27,7 +26,7 @@ url = 'https://www.flickr.com/'
2726search_url = url + 'search?{query}&page={page}'
2827time_range_url = '&min_upload_date={start}&max_upload_date={end}'
2928photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
30−regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
29+modelexport_re = re.compile(r"^\s*modelExport:\s*({.*}),$", re.M)
3130image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
3231
3332paging = True
@@ -57,40 +56,45 @@ def request(query, params):
5756def response(resp):
5857 results = []
5958
60− matches = regex.search(resp.text)
59+ matches = modelexport_re.search(resp.text)
6160
6261 if matches is None:
6362 return results
6463
6564 match = matches.group(1)
66− search_results = loads(match)
65+ model_export = loads(match)
6766
68− if '_data' not in search_results:
69− return []
67+ if 'legend' not in model_export:
68+ return results
7069
71− photos = search_results['_data']
70+ legend = model_export['legend']
7271
73− for photo in photos:
72+ # handle empty page
73+ if not legend or not legend[0]:
74+ return results
7475
75− # In paged configuration, the first pages' photos
76− # are represented by a None object
77− if photo is None:
78− continue
76+ for index in legend:
77+ photo = model_export['main'][index[0]][int(index[1])][index[2]][index[3]][int(index[4])]
78+ author = unquote(photo.get('realname', ''))
79+ source = unquote(photo.get('username', '')) + ' @ Flickr'
80+ title = unquote(photo.get('title', ''))
81+ content = unquote(photo.get('description', ''))
7982
8083 img_src = None
8184 # From the biggest to the lowest format
8285 for image_size in image_sizes:
8386 if image_size in photo['sizes']:
8487 img_src = photo['sizes'][image_size]['url']
88+ img_format = 'jpg ' \
89+ + str(photo['sizes'][image_size]['width']) \
90+ + 'x' \
91+ + str(photo['sizes'][image_size]['height'])
8592 break
8693
8794 if not img_src:
8895 logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
8996 continue
9097
91− if 'ownerNsid' not in photo:
92− continue
93−
9498 # For a bigger thumbnail, keep only the url_z, not the url_n
9599 if 'n' in photo['sizes']:
96100 thumbnail_src = photo['sizes']['n']['url']
@@ -99,19 +103,20 @@ def response(resp):
99103 else:
100104 thumbnail_src = img_src
101105
102− url = build_flickr_url(photo['ownerNsid'], photo['id'])
103−
104− title = photo.get('title', '')
105−
106− author = photo['username']
106+ if 'ownerNsid' not in photo:
107+ # should not happen, disowned photo? Show it anyway
108+ url = img_src
109+ else:
110+ url = build_flickr_url(photo['ownerNsid'], photo['id'])
107111
108− # append result
109112 results.append({'url': url,
110113 'title': title,
111114 'img_src': img_src,
112115 'thumbnail_src': thumbnail_src,
113− 'content': '',
116+ 'content': content,
114117 'author': author,
118+ 'source': source,
119+ 'img_format': img_format,
115120 'template': 'images.html'})
116121
117122 return results
tests/unit/engines/test_flickr_noapi.py+249−221
@@ -27,116 +27,132 @@ class TestFlickrNoapiEngine(SearxTestCase):
2727 self.assertRaises(AttributeError, flickr_noapi.response, '')
2828 self.assertRaises(AttributeError, flickr_noapi.response, '[]')
2929
30− response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":')
30+ response = mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{}}]}}')
3131 self.assertEqual(flickr_noapi.response(response), [])
3232
33− response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":')
33+ response = \
34+ mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{"_data":[]}}]}}')
3435 self.assertEqual(flickr_noapi.response(response), [])
3536
3637 # everthing is ok test
3738 json = """
38− "search-photos-lite-models","photos":
39− {
40− "_data": [
41− {
42− "_flickrModelRegistry": "photo-lite-models",
43− "title": "This is the title",
44− "username": "Owner",
45− "pathAlias": "klink692",
46− "realname": "Owner",
47− "license": 0,
48− "ownerNsid": "59729010@N00",
49− "canComment": false,
50− "commentCount": 14,
51− "faveCount": 21,
52− "id": "14001294434",
53− "sizes": {
54− "c": {
55− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg",
56− "width": 541,
57− "height": 800,
58− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg",
59− "key": "c"
60− },
61− "h": {
62− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg",
63− "width": 1081,
64− "height": 1600,
65− "url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg",
66− "key": "h"
67− },
68− "k": {
69− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg",
70− "width": 1383,
71− "height": 2048,
72− "url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg",
73− "key": "k"
74− },
75− "l": {
76− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg",
77− "width": 692,
78− "height": 1024,
79− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg",
80− "key": "l"
81− },
82− "m": {
83− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg",
84− "width": 338,
85− "height": 500,
86− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg",
87− "key": "m"
88− },
89− "n": {
90− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg",
91− "width": 216,
92− "height": 320,
93− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg",
94− "key": "n"
95− },
96− "q": {
97− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg",
98− "width": 150,
99− "height": 150,
100− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg",
101− "key": "q"
102− },
103− "s": {
104− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg",
105− "width": 162,
106− "height": 240,
107− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg",
108− "key": "s"
109− },
110− "sq": {
111− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg",
112− "width": 75,
113− "height": 75,
114− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg",
115− "key": "sq"
116− },
117− "t": {
118− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg",
119− "width": 68,
120− "height": 100,
121− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg",
122− "key": "t"
123− },
124− "z": {
125− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
126− "width": 433,
127− "height": 640,
128− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
129− "key": "z"
39+ modelExport: {
40+ "legend": [
41+ [
42+ "search-photos-lite-models",
43+ "0",
44+ "photos",
45+ "_data",
46+ "0"
47+ ]
48+ ],
49+ "main": {
50+ "search-photos-lite-models": [
51+ {
52+ "photos": {
53+ "_data": [
54+ {
55+ "_flickrModelRegistry": "photo-lite-models",
56+ "title": "This%20is%20the%20title",
57+ "username": "Owner",
58+ "pathAlias": "klink692",
59+ "realname": "Owner",
60+ "license": 0,
61+ "ownerNsid": "59729010@N00",
62+ "canComment": false,
63+ "commentCount": 14,
64+ "faveCount": 21,
65+ "id": "14001294434",
66+ "sizes": {
67+ "c": {
68+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg",
69+ "width": 541,
70+ "height": 800,
71+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg",
72+ "key": "c"
73+ },
74+ "h": {
75+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg",
76+ "width": 1081,
77+ "height": 1600,
78+ "url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg",
79+ "key": "h"
80+ },
81+ "k": {
82+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg",
83+ "width": 1383,
84+ "height": 2048,
85+ "url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg",
86+ "key": "k"
87+ },
88+ "l": {
89+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg",
90+ "width": 692,
91+ "height": 1024,
92+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg",
93+ "key": "l"
94+ },
95+ "m": {
96+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg",
97+ "width": 338,
98+ "height": 500,
99+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg",
100+ "key": "m"
101+ },
102+ "n": {
103+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg",
104+ "width": 216,
105+ "height": 320,
106+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg",
107+ "key": "n"
108+ },
109+ "q": {
110+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg",
111+ "width": 150,
112+ "height": 150,
113+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg",
114+ "key": "q"
115+ },
116+ "s": {
117+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg",
118+ "width": 162,
119+ "height": 240,
120+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg",
121+ "key": "s"
122+ },
123+ "sq": {
124+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg",
125+ "width": 75,
126+ "height": 75,
127+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg",
128+ "key": "sq"
129+ },
130+ "t": {
131+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg",
132+ "width": 68,
133+ "height": 100,
134+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg",
135+ "key": "t"
136+ },
137+ "z": {
138+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
139+ "width": 433,
140+ "height": 640,
141+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
142+ "key": "z"
143+ }
144+ }
145+ }
146+ ]
130147 }
131148 }
132− }
133− ],
134− "fetchedStart": true,
135− "fetchedEnd": false,
136− "totalItems": "4386039"
137− },"totalItems":
149+ ]
150+ }
151+ }
138152 """
139− json = json.replace('\r\n', '').replace('\n', '').replace('\r', '')
153+ # Flickr serves search results in a json block named 'modelExport' buried inside a script tag,
154+ # this json is served as a single line terminating with a comma.
155+ json = ''.join(json.split()) + ',\n'
140156 response = mock.Mock(text=json)
141157 results = flickr_noapi.response(response)
142158 self.assertEqual(type(results), list)
@@ -149,37 +165,51 @@ class TestFlickrNoapiEngine(SearxTestCase):
149165
150166 # no n size, only the z size
151167 json = """
152− "search-photos-lite-models","photos":
153− {
154− "_data": [
155− {
156− "_flickrModelRegistry": "photo-lite-models",
157− "title": "This is the title",
158− "username": "Owner",
159− "pathAlias": "klink692",
160− "realname": "Owner",
161− "license": 0,
162− "ownerNsid": "59729010@N00",
163− "canComment": false,
164− "commentCount": 14,
165− "faveCount": 21,
166− "id": "14001294434",
167− "sizes": {
168− "z": {
169− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
170− "width": 433,
171− "height": 640,
172− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
173− "key": "z"
168+ modelExport: {
169+ "legend": [
170+ [
171+ "search-photos-lite-models",
172+ "0",
173+ "photos",
174+ "_data",
175+ "0"
176+ ]
177+ ],
178+ "main": {
179+ "search-photos-lite-models": [
180+ {
181+ "photos": {
182+ "_data": [
183+ {
184+ "_flickrModelRegistry": "photo-lite-models",
185+ "title": "This%20is%20the%20title",
186+ "username": "Owner",
187+ "pathAlias": "klink692",
188+ "realname": "Owner",
189+ "license": 0,
190+ "ownerNsid": "59729010@N00",
191+ "canComment": false,
192+ "commentCount": 14,
193+ "faveCount": 21,
194+ "id": "14001294434",
195+ "sizes": {
196+ "z": {
197+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg",
198+ "width": 433,
199+ "height": 640,
200+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg",
201+ "key": "z"
202+ }
203+ }
204+ }
205+ ]
174206 }
175207 }
176− }
177− ],
178− "fetchedStart": true,
179− "fetchedEnd": false,
180− "totalItems": "4386039"
181− },"totalItems":
208+ ]
209+ }
210+ }
182211 """
212+ json = ''.join(json.split()) + ',\n'
183213 response = mock.Mock(text=json)
184214 results = flickr_noapi.response(response)
185215 self.assertEqual(type(results), list)
@@ -192,37 +222,51 @@ class TestFlickrNoapiEngine(SearxTestCase):
192222
193223 # no z or n size
194224 json = """
195− "search-photos-lite-models","photos":
196− {
197− "_data": [
198− {
199− "_flickrModelRegistry": "photo-lite-models",
200− "title": "This is the title",
201− "username": "Owner",
202− "pathAlias": "klink692",
203− "realname": "Owner",
204− "license": 0,
205− "ownerNsid": "59729010@N00",
206− "canComment": false,
207− "commentCount": 14,
208− "faveCount": 21,
209− "id": "14001294434",
210− "sizes": {
211− "o": {
212− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
213− "width": 433,
214− "height": 640,
215− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
216− "key": "o"
225+ modelExport: {
226+ "legend": [
227+ [
228+ "search-photos-lite-models",
229+ "0",
230+ "photos",
231+ "_data",
232+ "0"
233+ ]
234+ ],
235+ "main": {
236+ "search-photos-lite-models": [
237+ {
238+ "photos": {
239+ "_data": [
240+ {
241+ "_flickrModelRegistry": "photo-lite-models",
242+ "title": "This%20is%20the%20title",
243+ "username": "Owner",
244+ "pathAlias": "klink692",
245+ "realname": "Owner",
246+ "license": 0,
247+ "ownerNsid": "59729010@N00",
248+ "canComment": false,
249+ "commentCount": 14,
250+ "faveCount": 21,
251+ "id": "14001294434",
252+ "sizes": {
253+ "o": {
254+ "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
255+ "width": 433,
256+ "height": 640,
257+ "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
258+ "key": "o"
259+ }
260+ }
261+ }
262+ ]
217263 }
218264 }
219− }
220− ],
221− "fetchedStart": true,
222− "fetchedEnd": false,
223− "totalItems": "4386039"
224− },"totalItems":
265+ ]
266+ }
267+ }
225268 """
269+ json = ''.join(json.split()) + ',\n'
226270 response = mock.Mock(text=json)
227271 results = flickr_noapi.response(response)
228272 self.assertEqual(type(results), list)
@@ -235,30 +279,44 @@ class TestFlickrNoapiEngine(SearxTestCase):
235279
236280 # no image test
237281 json = """
238− "search-photos-lite-models","photos":
239− {
240− "_data": [
241− {
242− "_flickrModelRegistry": "photo-lite-models",
243− "title": "This is the title",
244− "username": "Owner",
245− "pathAlias": "klink692",
246− "realname": "Owner",
247− "license": 0,
248− "ownerNsid": "59729010@N00",
249− "canComment": false,
250− "commentCount": 14,
251− "faveCount": 21,
252− "id": "14001294434",
253− "sizes": {
254− }
255− }
282+ modelExport: {
283+ "legend": [
284+ [
285+ "search-photos-lite-models",
286+ "0",
287+ "photos",
288+ "_data",
289+ "0"
290+ ]
256291 ],
257− "fetchedStart": true,
258− "fetchedEnd": false,
259− "totalItems": "4386039"
260− },"totalItems":
292+ "main": {
293+ "search-photos-lite-models": [
294+ {
295+ "photos": {
296+ "_data": [
297+ {
298+ "_flickrModelRegistry": "photo-lite-models",
299+ "title": "This is the title",
300+ "username": "Owner",
301+ "pathAlias": "klink692",
302+ "realname": "Owner",
303+ "license": 0,
304+ "ownerNsid": "59729010@N00",
305+ "canComment": false,
306+ "commentCount": 14,
307+ "faveCount": 21,
308+ "id": "14001294434",
309+ "sizes": {
310+ }
311+ }
312+ ]
313+ }
314+ }
315+ ]
316+ }
317+ }
261318 """
319+ json = ''.join(json.split()) + ',\n'
262320 response = mock.Mock(text=json)
263321 results = flickr_noapi.response(response)
264322 self.assertEqual(type(results), list)
@@ -266,51 +324,20 @@ class TestFlickrNoapiEngine(SearxTestCase):
266324
267325 # null test
268326 json = """
269− "search-photos-models","photos":
270− {
271− "_data": [null],
272− "fetchedStart": true,
273− "fetchedEnd": false,
274− "totalItems": "4386039"
275− },"totalItems":
276− """
277− response = mock.Mock(text=json)
278− results = flickr_noapi.response(response)
279− self.assertEqual(type(results), list)
280− self.assertEqual(len(results), 0)
281−
282− # no ownerNsid test
283− json = """
284− "search-photos-lite-models","photos":
285− {
286− "_data": [
287− {
288− "_flickrModelRegistry": "photo-lite-models",
289− "title": "This is the title",
290− "username": "Owner",
291− "pathAlias": "klink692",
292− "realname": "Owner",
293− "license": 0,
294− "canComment": false,
295− "commentCount": 14,
296− "faveCount": 21,
297− "id": "14001294434",
298− "sizes": {
299− "o": {
300− "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg",
301− "width": 433,
302− "height": 640,
303− "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg",
304− "key": "o"
327+ modelExport: {
328+ "legend": [null],
329+ "main": {
330+ "search-photos-lite-models": [
331+ {
332+ "photos": {
333+ "_data": [null]
305334 }
306335 }
307− }
308− ],
309− "fetchedStart": true,
310− "fetchedEnd": false,
311− "totalItems": "4386039"
312− },"totalItems":
336+ ]
337+ }
338+ }
313339 """
340+ json = ''.join(json.split()) + ',\n'
314341 response = mock.Mock(text=json)
315342 results = flickr_noapi.response(response)
316343 self.assertEqual(type(results), list)
@@ -323,6 +350,7 @@ class TestFlickrNoapiEngine(SearxTestCase):
323350 "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"}
324351 ]}
325352 """
353+ json = ''.join(json.split()) + ',\n'
326354 response = mock.Mock(text=json)
327355 results = flickr_noapi.response(response)
328356 self.assertEqual(type(results), list)