git.iko.soy/searx
on master help sign in
Flake8 and Twitter corrections

Lots of Flake8 corrections Maybe we should change the rule to allow lines of 120 chars. It seems more usable. Big twitter correction : now it outputs the words in right order...

commit5d977056f7aa216eae09a22c3baaff73546f6ff1 authorCqoicebordel <Cqoicebordel@users.noreply.github.com> dateMon 29 Dec 2014, 21:31 (12 years ago) parent576fdef
13 files changed+80−47
compare with master
unifiedsplit
searx/engines/500px.py+2−2 searx/engines/__init__.py+2−2 searx/engines/digg.py+2−1 searx/engines/flickr-noapi.py+9−3 searx/engines/flickr.py+19−12 searx/engines/kickass.py+3−2 searx/engines/searchcode_code.py+9−9 searx/engines/searchcode_doc.py+11−4 searx/engines/subtitleseeker.py+6−2 searx/engines/twitter.py+4−3 searx/https_rewrite.py−1 searx/search.py+10−4 searx/utils.py+3−2
searx/engines/500px.py+2−2
@@ -35,9 +35,9 @@ def request(query, params):
3535# get response from search-request
3636def response(resp):
3737 results = []
38−
38+
3939 dom = html.fromstring(resp.text)
40−
40+
4141 # parse results
4242 for result in dom.xpath('//div[@class="photo"]'):
4343 link = result.xpath('.//a')[0]
searx/engines/__init__.py+2−2
@@ -81,7 +81,7 @@ def load_engine(engine_data):
8181 if engine_attr.startswith('_'):
8282 continue
8383 if getattr(engine, engine_attr) is None:
84− print('[E] Engine config error: Missing attribute "{0}.{1}"'\
84+ print('[E] Engine config error: Missing attribute "{0}.{1}"'
8585 .format(engine.name, engine_attr))
8686 sys.exit(1)
8787
@@ -102,7 +102,7 @@ def load_engine(engine_data):
102102 if engine.shortcut:
103103 # TODO check duplications
104104 if engine.shortcut in engine_shortcuts:
105− print('[E] Engine config error: ambigious shortcut: {0}'\
105+ print('[E] Engine config error: ambigious shortcut: {0}'
106106 .format(engine.shortcut))
107107 sys.exit(1)
108108 engine_shortcuts[engine.shortcut] = engine.name
searx/engines/digg.py+2−1
@@ -52,7 +52,8 @@ def response(resp):
5252 thumbnail = result.xpath('.//img')[0].attrib.get('src')
5353 title = ''.join(result.xpath(title_xpath))
5454 content = escape(''.join(result.xpath(content_xpath)))
55− publishedDate = parser.parse(result.xpath(pubdate_xpath)[0].attrib.get('datetime'))
55+ pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')
56+ publishedDate = parser.parse(pubdate)
5657
5758 # append result
5859 results.append({'url': url,
searx/engines/flickr-noapi.py+9−3
@@ -53,7 +53,8 @@ def response(resp):
5353
5454 for photo in photos:
5555
56− # In paged configuration, the first pages' photos are represented by a None object
56+ # In paged configuration, the first pages' photos
57+ # are represented by a None object
5758 if photo is None:
5859 continue
5960
@@ -74,10 +75,15 @@ def response(resp):
7475
7576 title = photo['title']
7677
77− content = '<span class="photo-author">' + photo['owner']['username'] + '</span><br />'
78+ content = '<span class="photo-author">' +\
79+ photo['owner']['username'] +\
80+ '</span><br />'
7881
7982 if 'description' in photo:
80− content = content + '<span class="description">' + photo['description'] + '</span>'
83+ content = content +\
84+ '<span class="description">' +\
85+ photo['description'] +\
86+ '</span>'
8187
8288 # append result
8389 results.append({'url': url,
searx/engines/flickr.py+19−12
@@ -1,10 +1,10 @@
11#!/usr/bin/env python
22
33## Flickr (Images)
4−#
4+#
55# @website https://www.flickr.com
6−# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)
7−#
6+# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)
7+#
88# @using-api yes
99# @results JSON
1010# @stable yes
@@ -18,16 +18,20 @@ categories = ['images']
1818
1919nb_per_page = 15
2020paging = True
21−api_key= None
21+api_key = None
2222
2323
24−url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key={api_key}&{text}&sort=relevance&extras=description%2C+owner_name%2C+url_o%2C+url_z&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}'
24+url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search' +\
25+ '&api_key={api_key}&{text}&sort=relevance' +\
26+ '&extras=description%2C+owner_name%2C+url_o%2C+url_z' +\
27+ '&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}'
2528photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
2629
2730paging = True
2831
32+
2933def build_flickr_url(user_id, photo_id):
30− return photo_url.format(userid=user_id,photoid=photo_id)
34+ return photo_url.format(userid=user_id, photoid=photo_id)
3135
3236
3337def request(query, params):
@@ -40,7 +44,7 @@ def request(query, params):
4044
4145def response(resp):
4246 results = []
43−
47+
4448 search_results = loads(resp.text)
4549
4650 # return empty array if there are no results
@@ -64,11 +68,14 @@ def response(resp):
6468 url = build_flickr_url(photo['owner'], photo['id'])
6569
6670 title = photo['title']
67−
68− content = '<span class="photo-author">'+ photo['ownername'] +'</span><br />'
69−
70− content = content + '<span class="description">' + photo['description']['_content'] + '</span>'
71−
71+
72+ content = '<span class="photo-author">' +\
73+ photo['ownername'] +\
74+ '</span><br />' +\
75+ '<span class="description">' +\
76+ photo['description']['_content'] +\
77+ '</span>'
78+
7279 # append result
7380 results.append({'url': url,
7481 'title': title,
searx/engines/kickass.py+3−2
@@ -24,7 +24,7 @@ search_url = url + 'search/{search_term}/{pageno}/'
2424
2525# specific xpath variables
2626magnet_xpath = './/a[@title="Torrent magnet link"]'
27−#content_xpath = './/font[@class="detDesc"]//text()'
27+content_xpath = './/span[@class="font11px lightgrey block"]'
2828
2929
3030# do search-request
@@ -56,7 +56,8 @@ def response(resp):
5656 link = result.xpath('.//a[@class="cellMainLink"]')[0]
5757 href = urljoin(url, link.attrib['href'])
5858 title = ' '.join(link.xpath('.//text()'))
59− content = escape(html.tostring(result.xpath('.//span[@class="font11px lightgrey block"]')[0], method="text"))
59+ content = escape(html.tostring(result.xpath(content_xpath)[0],
60+ method="text"))
6061 seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
6162 leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
6263
searx/engines/searchcode_code.py+9−9
@@ -11,7 +11,6 @@
1111from urllib import urlencode
1212from json import loads
1313import cgi
14−import re
1514
1615# engine dependent config
1716categories = ['it']
@@ -33,7 +32,7 @@ def request(query, params):
3332# get response from search-request
3433def response(resp):
3534 results = []
36−
35+
3736 search_results = loads(resp.text)
3837
3938 # parse results
@@ -41,21 +40,22 @@ def response(resp):
4140 href = result['url']
4241 title = "" + result['name'] + " - " + result['filename']
4342 content = result['repo'] + "<br />"
44−
43+
4544 lines = dict()
4645 for line, code in result['lines'].items():
4746 lines[int(line)] = code
4847
4948 content = content + '<pre class="code-formatter"><table class="code">'
5049 for line, code in sorted(lines.items()):
51− content = content + '<tr><td class="line-number" style="padding-right:5px;">'
52− content = content + str(line) + '</td><td class="code-snippet">'
53− # Replace every two spaces with ' &nbps;' to keep formatting while allowing the browser to break the line if necessary
54− content = content + cgi.escape(code).replace('\t', ' ').replace(' ', '&nbsp; ').replace(' ', ' &nbsp;')
50+ content = content + '<tr><td class="line-number" style="padding-right:5px;">'
51+ content = content + str(line) + '</td><td class="code-snippet">'
52+ # Replace every two spaces with ' &nbps;' to keep formatting
53+ # while allowing the browser to break the line if necessary
54+ content = content + cgi.escape(code).replace('\t', ' ').replace(' ', '&nbsp; ').replace(' ', ' &nbsp;')
5555 content = content + "</td></tr>"
56−
56+
5757 content = content + "</table></pre>"
58−
58+
5959 # append result
6060 results.append({'url': href,
6161 'title': title,
searx/engines/searchcode_doc.py+11−4
@@ -31,15 +31,22 @@ def request(query, params):
3131# get response from search-request
3232def response(resp):
3333 results = []
34−
34+
3535 search_results = loads(resp.text)
3636
3737 # parse results
3838 for result in search_results['results']:
3939 href = result['url']
40− title = "[" + result['type'] + "] " + result['namespace'] + " " + result['name']
41− content = '<span class="highlight">[' + result['type'] + "] " + result['name'] + " " + result['synopsis'] + "</span><br />" + result['description']
42−
40+ title = "[" + result['type'] + "] " +\
41+ result['namespace'] +\
42+ " " + result['name']
43+ content = '<span class="highlight">[' +\
44+ result['type'] + "] " +\
45+ result['name'] + " " +\
46+ result['synopsis'] +\
47+ "</span><br />" +\
48+ result['description']
49+
4350 # append result
4451 results.append({'url': href,
4552 'title': title,
searx/engines/subtitleseeker.py+6−2
@@ -60,10 +60,14 @@ def response(resp):
6060
6161 content = result.xpath('.//div[contains(@class,"red")]//text()')[0]
6262 content = content + " - "
63− content = content + html.tostring(result.xpath('.//div[contains(@class,"grey-web")]')[0], method='text')
63+ text = result.xpath('.//div[contains(@class,"grey-web")]')[0]
64+ content = content + html.tostring(text, method='text')
6465
6566 if result.xpath(".//span") != []:
66− content = content + " - (" + result.xpath(".//span//text()")[0].strip() + ")"
67+ content = content +\
68+ " - (" +\
69+ result.xpath(".//span//text()")[0].strip() +\
70+ ")"
6771
6872 # append result
6973 results.append({'url': href,
searx/engines/twitter.py+4−3
@@ -28,7 +28,7 @@ search_url = base_url+'search?'
2828results_xpath = '//li[@data-item-type="tweet"]'
2929link_xpath = './/small[@class="time"]//a'
3030title_xpath = './/span[@class="username js-action-profile-name"]//text()'
31−content_xpath = './/p[@class="js-tweet-text tweet-text"]//text()'
31+content_xpath = './/p[@class="js-tweet-text tweet-text"]'
3232timestamp_xpath = './/span[contains(@class,"_timestamp")]'
3333
3434
@@ -54,10 +54,11 @@ def response(resp):
5454 link = tweet.xpath(link_xpath)[0]
5555 url = urljoin(base_url, link.attrib.get('href'))
5656 title = ''.join(tweet.xpath(title_xpath))
57− content = escape(''.join(tweet.xpath(content_xpath)))
57+ content = escape(html.tostring(tweet.xpath(content_xpath)[0], method='text', encoding='UTF-8').decode("utf-8"))
5858 pubdate = tweet.xpath(timestamp_xpath)
5959 if len(pubdate) > 0:
60− publishedDate = datetime.fromtimestamp(float(pubdate[0].attrib.get('data-time')), None)
60+ timestamp = float(pubdate[0].attrib.get('data-time'))
61+ publishedDate = datetime.fromtimestamp(timestamp, None)
6162 # append result
6263 results.append({'url': url,
6364 'title': title,
searx/https_rewrite.py−1
@@ -154,7 +154,6 @@ def load_https_rules(rules_path):
154154 print(' * {n} https-rules loaded'.format(n=len(https_rules)))
155155
156156
157−
158157def https_url_rewrite(result):
159158 skip_https_rewrite = False
160159 # check if HTTPS rewrite is possible
searx/search.py+10−4
@@ -69,11 +69,16 @@ def threaded_requests(requests):
6969 print('engine timeout: {0}'.format(th._engine_name))
7070
7171
72−
7372# get default reqest parameter
7473def default_request_params():
7574 return {
76− 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}, 'verify': True}
75+ 'method': 'GET',
76+ 'headers': {},
77+ 'data': {},
78+ 'url': '',
79+ 'cookies': {},
80+ 'verify': True
81+ }
7782
7883
7984# create a callback wrapper for the search engine results
@@ -487,14 +492,15 @@ class Search(object):
487492 continue
488493
489494 # append request to list
490− requests.append((req, request_params['url'], request_args, selected_engine['name']))
495+ requests.append((req, request_params['url'],
496+ request_args,
497+ selected_engine['name']))
491498
492499 if not requests:
493500 return results, suggestions, answers, infoboxes
494501 # send all search-request
495502 threaded_requests(requests)
496503
497−
498504 while not results_queue.empty():
499505 engine_name, engine_results = results_queue.get_nowait()
500506
searx/utils.py+3−2
@@ -30,8 +30,9 @@ def gen_useragent():
3030
3131
3232def searx_useragent():
33− return 'searx/{searx_version} {suffix}'.format(searx_version=VERSION_STRING,
34− suffix=settings['server'].get('useragent_suffix', ''))
33+ return 'searx/{searx_version} {suffix}'.format(
34+ searx_version=VERSION_STRING,
35+ suffix=settings['server'].get('useragent_suffix', ''))
3536
3637
3738def highlight_content(content, query):