@@ -1,7 +1,7 @@
11"""
22 Digg (News, Social media)
33
4− @website https://digg.com/
4+ @website https://digg.com
55 @provide-api no
66
77 @using-api no
@@ -11,59 +11,56 @@
1111"""
1212# pylint: disable=missing-function-docstring
1313
14−import random
15−import string
1614from json import loads
1715from urllib.parse import urlencode
1816from datetime import datetime
1917
18+from lxml import html
19+
2020# engine dependent config
2121categories = ['news', 'social media']
2222paging = True
23+base_url = 'https://digg.com'
2324
2425# search-url
25−base_url = 'https://digg.com/'
26−search_url = base_url + 'api/search/?{query}&from={position}&size=20&format=html'
27−
28−# specific xpath variables
29−results_xpath = '//article'
30−link_xpath = './/small[@class="time"]//a'
31−title_xpath = './/h2//a//text()'
32−content_xpath = './/p//text()'
33−pubdate_xpath = './/time'
34−
35−digg_cookie_chars = string.ascii_uppercase + string.ascii_lowercase +\
36− string.digits + "+_"
37−
26+search_url = base_url + (
27+ '/api/search/'
28+ '?{query}'
29+ '&from={position}'
30+ '&size=20'
31+ '&format=html'
32+)
3833
39−# do search-request
4034def request(query, params):
4135 offset = (params['pageno'] - 1) * 20
42− params['url'] = search_url.format(position=offset,
43− query=urlencode({'q': query}))
44− params['cookies']['frontend.auid'] = ''.join(random.choice(
45− digg_cookie_chars) for _ in range(22))
36+ params['url'] = search_url.format(
37+ query = urlencode({'q': query}),
38+ position = offset,
39+ )
4640 return params
4741
48−
49−# get response from search-request
5042def response(resp):
5143 results = []
5244
53− search_result = loads(resp.text)
54−
5545 # parse results
56− for result in search_result['mapped']:
46+ for result in loads(resp.text)['mapped']:
47+
48+ # strip html tags and superfluous quotation marks from content
49+ content = html.document_fromstring(
50+ result['excerpt']
51+ ).text_content()
5752
5853 # 'created': {'ISO': '2020-10-16T14:09:55Z', ...}
59− published = datetime.strptime(result['created']['ISO'], "%Y-%m-%dT%H:%M:%SZ")
60− # append result
61− results.append({'url': result['url'],
62− 'title': result['title'],
63− 'content': result['excerpt'],
64− 'template': 'videos.html',
65− 'publishedDate': published,
66− 'thumbnail': result['images']['thumbImage']})
54+ published = datetime.strptime(
55+ result['created']['ISO'], '%Y-%m-%dT%H:%M:%SZ'
56+ )
57+ results.append({
58+ 'url': result['url'],
59+ 'title': result['title'],
60+ 'content' : content,
61+ 'template': 'videos.html',
62+ 'publishedDate': published,
63+ 'thumbnail': result['images']['thumbImage'],
64+ })
6765
68− # return results
6966 return results