git.iko.soy/searx
on master help sign in
Merge pull request #1412 from d-tux/engines/unsplash

Unsplash image engine

commit3e6d5ba35e198ba7e41c25115caf1612d508402b authorNoémi Ványi <kvch@users.noreply.github.com> dateSat 19 Jan 2019, 12:01 (8 years ago) parent89dd3dde946350
4 files changed+336
compare with master
unifiedsplit
searx/engines/unsplash.py+52 searx/settings.yml+5 tests/unit/engines/test_unsplash.py+38 tests/unit/engines/unsplash_fixture.json+241
searx/engines/unsplash.pynew file+52
@@ -0,0 +1,52 @@
1+"""
2+ Unsplash
3+
4+ @website https://unsplash.com
5+ @provide-api yes (https://unsplash.com/developers)
6+
7+ @using-api no
8+ @results JSON (using search portal's infiniscroll API)
9+ @stable no (JSON format could change any time)
10+ @parse url, title, img_src, thumbnail_src
11+"""
12+
13+from searx.url_utils import urlencode, urlparse, urlunparse, parse_qsl
14+from json import loads
15+
16+url = 'https://unsplash.com/'
17+search_url = url + 'napi/search/photos?'
18+categories = ['images']
19+page_size = 20
20+paging = True
21+
22+
23+def clean_url(url):
24+ parsed = urlparse(url)
25+ query = [(k, v) for (k, v) in parse_qsl(parsed.query) if k not in ['ixid', 's']]
26+
27+ return urlunparse((parsed.scheme,
28+ parsed.netloc,
29+ parsed.path,
30+ parsed.params,
31+ urlencode(query),
32+ parsed.fragment))
33+
34+
35+def request(query, params):
36+ params['url'] = search_url + urlencode({'query': query, 'page': params['pageno'], 'per_page': page_size})
37+ return params
38+
39+
40+def response(resp):
41+ results = []
42+ json_data = loads(resp.text)
43+
44+ if 'results' in json_data:
45+ for result in json_data['results']:
46+ results.append({'template': 'images.html',
47+ 'url': clean_url(result['links']['html']),
48+ 'thumbnail_src': clean_url(result['urls']['thumb']),
49+ 'img_src': clean_url(result['urls']['raw']),
50+ 'title': result['description'],
51+ 'content': ''})
52+ return results
searx/settings.yml+5
@@ -616,6 +616,11 @@ engines:
616616# content_xpath : //*[@class="meaning"]
617617# shortcut : ud
618618
619+ - name : unsplash
620+ engine : unsplash
621+ disabled: True
622+ shortcut : us
623+
619624 - name : yahoo
620625 engine : yahoo
621626 shortcut : yh
tests/unit/engines/test_unsplash.pynew file+38
@@ -0,0 +1,38 @@
1+from collections import defaultdict
2+import mock
3+from searx.testing import SearxTestCase
4+from searx.engines import unsplash
5+
6+
7+class TestUnsplashEngine(SearxTestCase):
8+ def test_request(self):
9+ query = 'penguin'
10+ _dict = defaultdict(dict)
11+ _dict['pageno'] = 1
12+ params = unsplash.request(query, _dict)
13+
14+ self.assertTrue('url' in params)
15+ self.assertTrue(query in params['url'])
16+
17+ def test_response(self):
18+ resp = mock.Mock(text='{}')
19+ result = unsplash.response(resp)
20+ self.assertEqual([], result)
21+
22+ resp.text = '{"results": []}'
23+ result = unsplash.response(resp)
24+ self.assertEqual([], result)
25+
26+ # Sourced from https://unsplash.com/napi/search/photos?query=penguin&xp=&per_page=20&page=2
27+ with open('./tests/unit/engines/unsplash_fixture.json') as fixture:
28+ resp.text = fixture.read()
29+
30+ result = unsplash.response(resp)
31+ self.assertEqual(len(result), 2)
32+ self.assertEqual(result[0]['title'], 'low angle photography of swimming penguin')
33+ self.assertEqual(result[0]['url'], 'https://unsplash.com/photos/FY8d721UO_4')
34+ self.assertEqual(result[0]['thumbnail_src'], 'https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80\
35+&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max')
36+ self.assertEqual(result[0]['img_src'], 'https://images.unsplash.com/photo-1523557148507-1b77641c7e7c\
37+?ixlib=rb-0.3.5')
38+ self.assertEqual(result[0]['content'], '')
tests/unit/engines/unsplash_fixture.jsonnew file+241
@@ -0,0 +1,241 @@
1+{
2+ "total": 2,
3+ "total_pages": 1,
4+ "results": [
5+ {
6+ "id": "FY8d721UO_4",
7+ "created_at": "2018-04-12T14:20:35-04:00",
8+ "updated_at": "2018-08-28T20:58:33-04:00",
9+ "width": 3891,
10+ "height": 5829,
11+ "color": "#152C33",
12+ "description": "low angle photography of swimming penguin",
13+ "urls": {
14+ "raw": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&ixid=eyJhcHBfaWQiOjEyMDd9&s=095c5fc319c5a77c705f49ad63e0f195",
15+ "full": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=85&fm=jpg&crop=entropy&cs=srgb&ixid=eyJhcHBfaWQiOjEyMDd9&s=74be977849c173d6929636d491a760c3",
16+ "regular": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=1080&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=ad65df26970bd010085f0ca25434de33",
17+ "small": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=400&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=5d2edfd073c31eb8ee7b305222bdc5a2",
18+ "thumb": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=a9b9e56e63efc6f4611a87ce7e9a48f8"
19+ },
20+ "links": {
21+ "self": "https://api.unsplash.com/photos/FY8d721UO_4",
22+ "html": "https://unsplash.com/photos/FY8d721UO_4",
23+ "download": "https://unsplash.com/photos/FY8d721UO_4/download",
24+ "download_location": "https://api.unsplash.com/photos/FY8d721UO_4/download"
25+ },
26+ "categories": [],
27+ "sponsored": false,
28+ "likes": 31,
29+ "liked_by_user": false,
30+ "current_user_collections": [],
31+ "slug": null,
32+ "user": {
33+ "id": "N4gE4mrG8lE",
34+ "updated_at": "2018-10-03T02:51:19-04:00",
35+ "username": "gaspanik",
36+ "name": "Masaaki Komori",
37+ "first_name": "Masaaki",
38+ "last_name": "Komori",
39+ "twitter_username": "cipher",
40+ "portfolio_url": "https://www.instagram.com/cipher/",
41+ "bio": null,
42+ "location": "Tokyo, JAPAN",
43+ "links": {
44+ "self": "https://api.unsplash.com/users/gaspanik",
45+ "html": "https://unsplash.com/@gaspanik",
46+ "photos": "https://api.unsplash.com/users/gaspanik/photos",
47+ "likes": "https://api.unsplash.com/users/gaspanik/likes",
48+ "portfolio": "https://api.unsplash.com/users/gaspanik/portfolio",
49+ "following": "https://api.unsplash.com/users/gaspanik/following",
50+ "followers": "https://api.unsplash.com/users/gaspanik/followers"
51+ },
52+ "profile_image": {
53+ "small": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=32&w=32&s=9fe12f6d177bd6fdbd56d233a80c01a3",
54+ "medium": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=64&w=64&s=6ad7d156b62e438ae9dc794cba712fff",
55+ "large": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=128&w=128&s=13a08a2e72e7d11632410e92bd3a9406"
56+ },
57+ "instagram_username": "cipher",
58+ "total_collections": 0,
59+ "total_likes": 406,
60+ "total_photos": 196
61+ },
62+ "tags": [
63+ {
64+ "title": "animal"
65+ },
66+ {
67+ "title": "water"
68+ },
69+ {
70+ "title": "swim"
71+ },
72+ {
73+ "title": "aquarium"
74+ },
75+ {
76+ "title": "wallpaper"
77+ },
78+ {
79+ "title": "blue"
80+ },
81+ {
82+ "title": "sealife"
83+ },
84+ {
85+ "title": "wildlife"
86+ },
87+ {
88+ "title": "bird"
89+ },
90+ {
91+ "title": "deep sea"
92+ },
93+ {
94+ "title": "fish"
95+ },
96+ {
97+ "title": "water life"
98+ }
99+ ],
100+ "photo_tags": [
101+ {
102+ "title": "animal"
103+ },
104+ {
105+ "title": "water"
106+ },
107+ {
108+ "title": "swim"
109+ },
110+ {
111+ "title": "aquarium"
112+ },
113+ {
114+ "title": "wallpaper"
115+ }
116+ ]
117+ },
118+ {
119+ "id": "ayKyc01xLWA",
120+ "created_at": "2018-02-16T23:14:31-05:00",
121+ "updated_at": "2018-08-28T20:48:27-04:00",
122+ "width": 4928,
123+ "height": 3264,
124+ "color": "#161618",
125+ "description": "black and white penguins on ice field",
126+ "urls": {
127+ "raw": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&ixid=eyJhcHBfaWQiOjEyMDd9&s=4e107a2bc49ab561ba6272eea2ec725d",
128+ "full": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=85&fm=jpg&crop=entropy&cs=srgb&ixid=eyJhcHBfaWQiOjEyMDd9&s=f9b1e4d4572ab44efb2cf3d601d2b4d9",
129+ "regular": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=1080&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=4430cedb63841f1fe055d5005316cc96",
130+ "small": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=400&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=ee73c7af22ce445d408e240821ce07af",
131+ "thumb": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=934302390d383cad8c571905e3a80bac"
132+ },
133+ "links": {
134+ "self": "https://api.unsplash.com/photos/ayKyc01xLWA",
135+ "html": "https://unsplash.com/photos/ayKyc01xLWA",
136+ "download": "https://unsplash.com/photos/ayKyc01xLWA/download",
137+ "download_location": "https://api.unsplash.com/photos/ayKyc01xLWA/download"
138+ },
139+ "categories": [],
140+ "sponsored": false,
141+ "likes": 37,
142+ "liked_by_user": false,
143+ "current_user_collections": [],
144+ "slug": null,
145+ "user": {
146+ "id": "tRb_KGw60Xk",
147+ "updated_at": "2018-09-20T11:51:54-04:00",
148+ "username": "ghost_cat",
149+ "name": "Danielle Barnes",
150+ "first_name": "Danielle",
151+ "last_name": "Barnes",
152+ "twitter_username": null,
153+ "portfolio_url": null,
154+ "bio": null,
155+ "location": null,
156+ "links": {
157+ "self": "https://api.unsplash.com/users/ghost_cat",
158+ "html": "https://unsplash.com/@ghost_cat",
159+ "photos": "https://api.unsplash.com/users/ghost_cat/photos",
160+ "likes": "https://api.unsplash.com/users/ghost_cat/likes",
161+ "portfolio": "https://api.unsplash.com/users/ghost_cat/portfolio",
162+ "following": "https://api.unsplash.com/users/ghost_cat/following",
163+ "followers": "https://api.unsplash.com/users/ghost_cat/followers"
164+ },
165+ "profile_image": {
166+ "small": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=32&w=32&s=751bf6a557763648d52ffd7e60e79436",
167+ "medium": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=64&w=64&s=e46cd1c8713035f045130e1b093b981e",
168+ "large": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=128&w=128&s=352eabcf107c3ce95fe51a18485f116b"
169+ },
170+ "instagram_username": null,
171+ "total_collections": 0,
172+ "total_likes": 0,
173+ "total_photos": 21
174+ },
175+ "tags": [
176+ {
177+ "title": "ice"
178+ },
179+ {
180+ "title": "bird"
181+ },
182+ {
183+ "title": "ice field"
184+ },
185+ {
186+ "title": "iceberg"
187+ },
188+ {
189+ "title": "snow"
190+ },
191+ {
192+ "title": "frozen"
193+ },
194+ {
195+ "title": "animal"
196+ },
197+ {
198+ "title": "wildlife"
199+ },
200+ {
201+ "title": "wild"
202+ },
203+ {
204+ "title": "antarctica"
205+ },
206+ {
207+ "title": "sunshine"
208+ },
209+ {
210+ "title": "daylight"
211+ },
212+ {
213+ "title": "wilderness"
214+ },
215+ {
216+ "title": "south pole"
217+ },
218+ {
219+ "title": "flock"
220+ }
221+ ],
222+ "photo_tags": [
223+ {
224+ "title": "ice"
225+ },
226+ {
227+ "title": "bird"
228+ },
229+ {
230+ "title": "ice field"
231+ },
232+ {
233+ "title": "iceberg"
234+ },
235+ {
236+ "title": "snow"
237+ }
238+ ]
239+ }
240+ ]
241+}