git.iko.soy/searx
on master help sign in
New engine: Genius (lyrics)
2434c29woorst
searx/engines/genius.pynew file+89
compare with master
unifiedsplit
@@ -0,0 +1,89 @@
1+"""
2+Genius
3+
4+ @website https://www.genius.com/
5+ @provide-api yes (https://docs.genius.com/)
6+
7+ @using-api yes
8+ @results JSON
9+ @stable yes
10+ @parse url, title, content, thumbnail, publishedDate
11+"""
12+
13+from json import loads
14+from searx.url_utils import urlencode
15+from datetime import datetime
16+
17+# engine dependent config
18+categories = ['music']
19+paging = True
20+language_support = False
21+page_size = 5
22+
23+indicies = ['top_hit', 'song', 'lyric', 'artist', 'album', 'tag', 'video', 'article', 'user']
24+url = 'https://genius.com/api/'
25+search_url = url + 'search/{index}?{query}&page={pageno}&per_page={page_size}'
26+
27+
28+def request(query, params):
29+ params['url'] = search_url.format(query=urlencode({'q': query}),
30+ index='multi',
31+ page_size=page_size,
32+ pageno=params['pageno'])
33+ return params
34+
35+
36+def parse_lyric(hit):
37+ try:
38+ content = hit['highlights'][0]['value']
39+ except:
40+ content = None
41+ timestamp = hit['result']['lyrics_updated_at']
42+ result = {'url': hit['result']['url'],
43+ 'title': hit['result']['full_title'],
44+ 'content': content,
45+ 'thumbnail': hit['result']['song_art_image_thumbnail_url'],
46+ 'template': 'videos.html'}
47+ if timestamp:
48+ result.update({'publishedDate': datetime.fromtimestamp(timestamp)})
49+ return result
50+
51+
52+def parse_artist(hit):
53+ result = {'url': hit['result']['url'],
54+ 'title': hit['result']['name'],
55+ 'content': None,
56+ 'thumbnail': hit['result']['image_url'],
57+ 'template': 'videos.html'}
58+ return result
59+
60+
61+def parse_album(hit):
62+ result = {'url': hit['result']['url'],
63+ 'title': hit['result']['full_title'],
64+ 'thumbnail': hit['result']['cover_art_url'],
65+ # 'thumbnail': hit['result']['cover_art_thumbnail_url'],
66+ 'template': 'videos.html'}
67+ try:
68+ year = hit['result']['release_date_components']['year']
69+ except:
70+ pass
71+ else:
72+ if year:
73+ result.update({'content': 'Released: {}'.format(year)})
74+ return result
75+
76+parse = {'lyric': parse_lyric, 'song': parse_lyric, 'artist': parse_artist, 'album': parse_album}
77+
78+
79+def response(resp):
80+ results = []
81+ json = loads(resp.text)
82+ hits = [hit for section in json['response']['sections'] for hit in section['hits']]
83+ for hit in hits:
84+ try:
85+ func = parse[hit['type']]
86+ except KeyError:
87+ continue
88+ results.append(func(hit))
89+ return results
searx/settings.yml+4
@@ -236,6 +236,10 @@ engines:
236 shortcut : frk
236 shortcut : frk
237 disabled : True
237 disabled : True
238
238
239+ - name : genius
240+ engine : genius
241+ shortcut : gen
242+
239 - name : gigablast
243 - name : gigablast
240 engine : gigablast
244 engine : gigablast
241 shortcut : gb
245 shortcut : gb
tests/unit/engines/test_genius.pynew file+231
@@ -0,0 +1,231 @@
1+from collections import defaultdict
2+import mock
3+from datetime import datetime
4+from searx.engines import genius
5+from searx.testing import SearxTestCase
6+
7+
8+class TestGeniusEngine(SearxTestCase):
9+
10+ def test_request(self):
11+ query = 'test_query'
12+ dicto = defaultdict(dict)
13+ dicto['pageno'] = 1
14+ params = genius.request(query, dicto)
15+ self.assertTrue('url' in params)
16+ self.assertTrue(query in params['url'])
17+ self.assertTrue('genius.com' in params['url'])
18+
19+ def test_response(self):
20+
21+ json_empty = """
22+ {
23+ "meta": {
24+ "status": 200
25+ },
26+ "response": {
27+ "sections": [
28+ {
29+ "type": "top_hit",
30+ "hits": []
31+ },
32+ {
33+ "type": "song",
34+ "hits": []
35+ },
36+ {
37+ "type": "lyric",
38+ "hits": []
39+ },
40+ {
41+ "type": "artist",
42+ "hits": []
43+ },
44+ {
45+ "type": "album",
46+ "hits": []
47+ },
48+ {
49+ "type": "tag",
50+ "hits": []
51+ },
52+ {
53+ "type": "video",
54+ "hits": []
55+ },
56+ {
57+ "type": "article",
58+ "hits": []
59+ },
60+ {
61+ "type": "user",
62+ "hits": []
63+ }
64+ ]
65+ }
66+ }
67+ """
68+
69+ resp = mock.Mock(text=json_empty)
70+ self.assertEqual(genius.response(resp), [])
71+
72+ json = """
73+ {
74+ "meta": {
75+ "status": 200
76+ },
77+ "response": {
78+ "sections": [
79+ {
80+ "type": "lyric",
81+ "hits": [
82+ {
83+ "highlights": [
84+ {
85+ "property": "lyrics",
86+ "value": "Sample lyrics",
87+ "snippet": true,
88+ "ranges": []
89+ }
90+ ],
91+ "index": "lyric",
92+ "type": "song",
93+ "result": {
94+ "_type": "song",
95+ "annotation_count": 45,
96+ "api_path": "/songs/52916",
97+ "full_title": "J't'emmerde by MC Jean Gab'1",
98+ "header_image_thumbnail_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.300x300x1.jpg",
99+ "header_image_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.1000x1000x1.jpg",
100+ "id": 52916,
101+ "instrumental": false,
102+ "lyrics_owner_id": 15586,
103+ "lyrics_state": "complete",
104+ "lyrics_updated_at": 1498744545,
105+ "path": "/Mc-jean-gab1-jtemmerde-lyrics",
106+ "pyongs_count": 4,
107+ "song_art_image_thumbnail_url": "https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.300x300x1.jpg",
108+ "stats": {
109+ "hot": false,
110+ "unreviewed_annotations": 0,
111+ "pageviews": 62490
112+ },
113+ "title": "J't'emmerde",
114+ "title_with_featured": "J't'emmerde",
115+ "updated_by_human_at": 1498744546,
116+ "url": "https://genius.com/Mc-jean-gab1-jtemmerde-lyrics",
117+ "primary_artist": {
118+ "_type": "artist",
119+ "api_path": "/artists/12691",
120+ "header_image_url": "https://images.genius.com/c7847662a58f8c2b0f02a6e217d60907.960x657x1.jpg",
121+ "id": 12691,
122+ "image_url": "https://s3.amazonaws.com/rapgenius/Mc-jean-gab1.jpg",
123+ "index_character": "m",
124+ "is_meme_verified": false,
125+ "is_verified": false,
126+ "name": "MC Jean Gab'1",
127+ "slug": "Mc-jean-gab1",
128+ "url": "https://genius.com/artists/Mc-jean-gab1"
129+ }
130+ }
131+ }
132+ ]
133+ },
134+ {
135+ "type": "artist",
136+ "hits": [
137+ {
138+ "highlights": [],
139+ "index": "artist",
140+ "type": "artist",
141+ "result": {
142+ "_type": "artist",
143+ "api_path": "/artists/191580",
144+ "header_image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
145+ "id": 191580,
146+ "image_url": "https://assets.genius.com/images/default_avatar_300.png?1503090542",
147+ "index_character": "a",
148+ "is_meme_verified": false,
149+ "is_verified": false,
150+ "name": "ASDF Guy",
151+ "slug": "Asdf-guy",
152+ "url": "https://genius.com/artists/Asdf-guy"
153+ }
154+ }
155+ ]
156+ },
157+ {
158+ "type": "album",
159+ "hits": [
160+ {
161+ "highlights": [],
162+ "index": "album",
163+ "type": "album",
164+ "result": {
165+ "_type": "album",
166+ "api_path": "/albums/132332",
167+ "cover_art_thumbnail_url": "https://images.genius.com/147d70434ba190b9b1c26b06aee87d17.300x300x1.jpg",
168+ "cover_art_url": "https://images.genius.com/147d70434ba190b9b1c26b06aee87d17.600x600x1.jpg",
169+ "full_title": "ASD by A Skylit Drive",
170+ "id": 132332,
171+ "name": "ASD",
172+ "name_with_artist": "ASD (artist: A Skylit Drive)",
173+ "release_date_components": {
174+ "year": 2015,
175+ "month": null,
176+ "day": null
177+ },
178+ "url": "https://genius.com/albums/A-skylit-drive/Asd",
179+ "artist": {
180+ "_type": "artist",
181+ "api_path": "/artists/48712",
182+ "header_image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
183+ "id": 48712,
184+ "image_url": "https://images.genius.com/814c1551293172c56306d0e310c6aa89.620x400x1.jpg",
185+ "index_character": "s",
186+ "is_meme_verified": false,
187+ "is_verified": false,
188+ "name": "A Skylit Drive",
189+ "slug": "A-skylit-drive",
190+ "url": "https://genius.com/artists/A-skylit-drive"
191+ }
192+ }
193+ }
194+ ]
195+ }
196+ ]
197+ }
198+ }
199+ """
200+
201+ resp = mock.Mock(text=json)
202+ results = genius.response(resp)
203+
204+ self.assertEqual(len(results), 3)
205+ self.assertEqual(type(results), list)
206+
207+ # check lyric parsing
208+ r = results[0]
209+ self.assertEqual(r['url'], 'https://genius.com/Mc-jean-gab1-jtemmerde-lyrics')
210+ self.assertEqual(r['title'], "J't'emmerde by MC Jean Gab'1")
211+ self.assertEqual(r['content'], "Sample lyrics")
212+ self.assertEqual(r['template'], 'videos.html')
213+ self.assertEqual(r['thumbnail'], 'https://images.genius.com/ef9f736a86df3c3b1772f3fb7fbdb21c.300x300x1.jpg')
214+ created = datetime.fromtimestamp(1498744545)
215+ self.assertEqual(r['publishedDate'], created)
216+
217+ # check artist parsing
218+ r = results[1]
219+ self.assertEqual(r['url'], 'https://genius.com/artists/Asdf-guy')
220+ self.assertEqual(r['title'], "ASDF Guy")
221+ self.assertEqual(r['content'], None)
222+ self.assertEqual(r['template'], 'videos.html')
223+ self.assertEqual(r['thumbnail'], 'https://assets.genius.com/images/default_avatar_300.png?1503090542')
224+
225+ # check album parsing
226+ r = results[2]
227+ self.assertEqual(r['url'], 'https://genius.com/albums/A-skylit-drive/Asd')
228+ self.assertEqual(r['title'], "ASD by A Skylit Drive")
229+ self.assertEqual(r['content'], "Released: 2015")
230+ self.assertEqual(r['template'], 'videos.html')
231+ self.assertEqual(r['thumbnail'], 'https://images.genius.com/147d70434ba190b9b1c26b06aee87d17.600x600x1.jpg')