@@ -7,75 +7,70 @@
77 @using-api no (TODO, rewrite to api)
88 @results HTML
99 @stable no (HTML can change)
10− @parse url, title, thumbnail_src, img_src
10+ @parse url, title, img_src
1111
1212 @todo rewrite to api
1313"""
14+# pylint: disable=missing-function-docstring
1415
15−from lxml import html
16−import re
1716from urllib.parse import urlencode
18−
17+from lxml import html
1918
2019# engine dependent config
2120categories = ['images']
2221paging = True
2322time_range_support = True
2423
25−# search-url
26−base_url = 'https://www.deviantart.com/'
27−search_url = base_url + 'search?page={page}&{query}'
28−time_range_url = '&order={range}'
29−
30−time_range_dict = {'day': 11,
31− 'week': 14,
32− 'month': 15}
24+time_range_dict = {
25+ 'day': 'popular-24-hours',
26+ 'week': 'popular-1-week',
27+ 'month': 'popular-1-month',
28+ 'year': 'most-recent',
29+}
3330
31+# search-url
32+base_url = 'https://www.deviantart.com'
3433
35−# do search-request
3634def request(query, params):
37− if params['time_range'] and params['time_range'] not in time_range_dict:
38− return params
3935
40− params['url'] = search_url.format(page=params['pageno'],
41− query=urlencode({'q': query}))
36+ # https://www.deviantart.com/search/deviations?page=5&q=foo
37+
38+ query = {
39+ 'page' : params['pageno'],
40+ 'q' : query,
41+ }
4242 if params['time_range'] in time_range_dict:
43− params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])
43+ query['order'] = time_range_dict[params['time_range']]
4444
45− return params
45+ params['url'] = base_url + '/search/deviations?' + urlencode(query)
4646
47+ return params
4748
48−# get response from search-request
4949def response(resp):
50− results = []
5150
52− # return empty array if a redirection code is returned
53− if resp.status_code == 302:
54− return []
51+ results = []
5552
5653 dom = html.fromstring(resp.text)
5754
58− # parse results
5955 for row in dom.xpath('//div[contains(@data-hook, "content_row")]'):
6056 for result in row.xpath('./div'):
61− link = result.xpath('.//a[@data-hook="deviation_link"]')[0]
62− url = link.attrib.get('href')
63− title = link.attrib.get('title')
64− thumbnail_src = result.xpath('.//img')[0].attrib.get('src')
65− img_src = thumbnail_src
66−
67− # http to https, remove domain sharding
68− thumbnail_src = re.sub(r"https?://(th|fc)\d+.", "https://th01.", thumbnail_src)
69− thumbnail_src = re.sub(r"http://", "https://", thumbnail_src)
70−
71− url = re.sub(r"http://(.*)\.deviantart\.com/", "https://\\1.deviantart.com/", url)
72−
73− # append result
74− results.append({'url': url,
75− 'title': title,
76− 'img_src': img_src,
77− 'thumbnail_src': thumbnail_src,
78− 'template': 'images.html'})
79−
80− # return results
57+
58+ a_tag = result.xpath('.//a[@data-hook="deviation_link"]')[0]
59+ noscript_tag = a_tag.xpath('.//noscript')
60+
61+ if noscript_tag:
62+ img_tag = noscript_tag[0].xpath('.//img')
63+ else:
64+ img_tag = a_tag.xpath('.//img')
65+ if not img_tag:
66+ continue
67+ img_tag = img_tag[0]
68+
69+ results.append({
70+ 'template': 'images.html',
71+ 'url': a_tag.attrib.get('href'),
72+ 'img_src': img_tag.attrib.get('src'),
73+ 'title': img_tag.attrib.get('alt'),
74+ })
75+
8176 return results