git.iko.soy/searx
on master help sign in
[refactor] digg - improve results and clean up source code
bef1857Markus Heiser
searx/engines/digg.py+32−35
compare with master
unifiedsplit
@@ -1,7 +1,7 @@
1"""
1"""
2 Digg (News, Social media)
2 Digg (News, Social media)
3
3
4− @website https://digg.com/
4+ @website https://digg.com
5 @provide-api no
5 @provide-api no
6
6
7 @using-api no
7 @using-api no
@@ -11,59 +11,56 @@
11"""
11"""
12# pylint: disable=missing-function-docstring
12# pylint: disable=missing-function-docstring
13
13
14−import random
15−import string
16from json import loads
14from json import loads
17from urllib.parse import urlencode
15from urllib.parse import urlencode
18from datetime import datetime
16from datetime import datetime
19
17
18+from lxml import html
19+
20# engine dependent config
20# engine dependent config
21categories = ['news', 'social media']
21categories = ['news', 'social media']
22paging = True
22paging = True
23+base_url = 'https://digg.com'
23
24
24# search-url
25# search-url
25−base_url = 'https://digg.com/'
26−search_url = base_url + 'api/search/?{query}&from={position}&size=20&format=html'
26+search_url = base_url + (
27−
27+ '/api/search/'
28−# specific xpath variables
28+ '?{query}'
29−results_xpath = '//article'
29+ '&from={position}'
30−link_xpath = './/small[@class="time"]//a'
30+ '&size=20'
31−title_xpath = './/h2//a//text()'
31+ '&format=html'
32−content_xpath = './/p//text()'
32+)
33−pubdate_xpath = './/time'
34−
35−digg_cookie_chars = string.ascii_uppercase + string.ascii_lowercase +\
36− string.digits + "+_"
37−
38
33
39−# do search-request
40def request(query, params):
34def request(query, params):
41 offset = (params['pageno'] - 1) * 20
35 offset = (params['pageno'] - 1) * 20
42− params['url'] = search_url.format(position=offset,
36+ params['url'] = search_url.format(
43− query=urlencode({'q': query}))
37+ query = urlencode({'q': query}),
44− params['cookies']['frontend.auid'] = ''.join(random.choice(
38+ position = offset,
45− digg_cookie_chars) for _ in range(22))
39+ )
46 return params
40 return params
47
41
48−
49−# get response from search-request
50def response(resp):
42def response(resp):
51 results = []
43 results = []
52
44
53− search_result = loads(resp.text)
54−
55 # parse results
45 # parse results
56− for result in search_result['mapped']:
46+ for result in loads(resp.text)['mapped']:
47+
48+ # strip html tags and superfluous quotation marks from content
49+ content = html.document_fromstring(
50+ result['excerpt']
51+ ).text_content()
57
52
58 # 'created': {'ISO': '2020-10-16T14:09:55Z', ...}
53 # 'created': {'ISO': '2020-10-16T14:09:55Z', ...}
59− published = datetime.strptime(result['created']['ISO'], "%Y-%m-%dT%H:%M:%SZ")
54+ published = datetime.strptime(
60− # append result
55+ result['created']['ISO'], '%Y-%m-%dT%H:%M:%SZ'
56+ )
61− results.append({'url': result['url'],
57+ results.append({
58+ 'url': result['url'],
62− 'title': result['title'],
59+ 'title': result['title'],
63− 'content': result['excerpt'],
60+ 'content' : content,
64− 'template': 'videos.html',
61+ 'template': 'videos.html',
65− 'publishedDate': published,
62+ 'publishedDate': published,
66− 'thumbnail': result['images']['thumbImage']})
63+ 'thumbnail': result['images']['thumbImage'],
64+ })
67
65
68− # return results
69 return results
66 return results