@@ -1,7 +1,7 @@
11"""
22 Digg (News, Social media)
33
4− @website https://digg.com/
4+ @website https://digg.com
55 @provide-api no
66
77 @using-api no
@@ -9,59 +9,58 @@
99 @stable no (HTML can change)
1010 @parse url, title, content, publishedDate, thumbnail
1111"""
12+# pylint: disable=missing-function-docstring
1213
13−import random
14−import string
1514from json import loads
1615from urllib.parse import urlencode
1716from datetime import datetime
1817
18+from lxml import html
19+
1920# engine dependent config
2021categories = ['news', 'social media']
2122paging = True
23+base_url = 'https://digg.com'
2224
2325# search-url
24−base_url = 'https://digg.com/'
25−search_url = base_url + 'api/search/?{query}&from={position}&size=20&format=html'
26−
27−# specific xpath variables
28−results_xpath = '//article'
29−link_xpath = './/small[@class="time"]//a'
30−title_xpath = './/h2//a//text()'
31−content_xpath = './/p//text()'
32−pubdate_xpath = './/time'
33−
34−digg_cookie_chars = string.ascii_uppercase + string.ascii_lowercase +\
35− string.digits + "+_"
36−
26+search_url = base_url + (
27+ '/api/search/'
28+ '?{query}'
29+ '&from={position}'
30+ '&size=20'
31+ '&format=html'
32+)
3733
38−# do search-request
3934def request(query, params):
4035 offset = (params['pageno'] - 1) * 20
41− params['url'] = search_url.format(position=offset,
42− query=urlencode({'q': query}))
43− params['cookies']['frontend.auid'] = ''.join(random.choice(
44− digg_cookie_chars) for _ in range(22))
36+ params['url'] = search_url.format(
37+ query = urlencode({'q': query}),
38+ position = offset,
39+ )
4540 return params
4641
47−
48−# get response from search-request
4942def response(resp):
5043 results = []
5144
52− search_result = loads(resp.text)
53−
5445 # parse results
55− for result in search_result['mapped']:
46+ for result in loads(resp.text)['mapped']:
47+
48+ # strip html tags and superfluous quotation marks from content
49+ content = html.document_fromstring(
50+ result['excerpt']
51+ ).text_content()
5652
57− published = datetime.strptime(result['created']['ISO'], "%Y-%m-%d %H:%M:%S")
58− # append result
59− results.append({'url': result['url'],
60− 'title': result['title'],
61− 'content': result['excerpt'],
62− 'template': 'videos.html',
63− 'publishedDate': published,
64− 'thumbnail': result['images']['thumbImage']})
53+ # 'created': {'ISO': '2020-10-16T14:09:55Z', ...}
54+ published = datetime.strptime(
55+ result['created']['ISO'], '%Y-%m-%dT%H:%M:%SZ'
56+ )
57+ results.append({
58+ 'url': result['url'],
59+ 'title': result['title'],
60+ 'content' : content,
61+ 'template': 'videos.html',
62+ 'publishedDate': published,
63+ 'thumbnail': result['images']['thumbImage'],
64+ })
6565
66− # return results
6766 return results