on master help sign in
[enh] https rewrite basics
commit96c8b20a045f62205a3b9a03113086f0fcfbc579 authorAdam Tauber <asciimoo@gmail.com> dateTue 24 Jun 2014, 16:30 (12 years ago) parentb446432
3 files changed+34−3
searx/https_rewrite.py+14 searx/settings.yml+2−1 searx/webapp.py+18−2
searx/https_rewrite.pynew file+14
@@ -0,0 +1,14 @@
1+import re
2+
3+# https://gitweb.torproject.org/\
4+# pde/https-everywhere.git/tree/4.0:/src/chrome/content/rules
5+
6+# HTTPS rewrite rules
7+https_rules = (
8+ # from
9+ (re.compile(r'^http://(www\.|m\.|)?xkcd\.(?:com|org)/', re.I | re.U),
10+ # to
11+ r'https://\1xkcd.com/'),
12+ (re.compile(r'^https?://(?:ssl)?imgs\.xkcd\.com/', re.I | re.U),
13+ r'https://sslimgs.xkcd.com/'),
14+)
searx/settings.yml+2−1
@@ -1,11 +1,12 @@
11server:
22 port : 8888
33 secret_key : "ultrasecretkey" # change this!
4− debug : True
4+ debug : False
55 request_timeout : 2.0 # seconds
66 base_url : False
77 themes_path : ""
88 default_theme : default
9+ https_rewrite : True
910
1011engines:
1112 - name : wikipedia
searx/webapp.py+18−2
@@ -41,13 +41,16 @@ from searx.engines import (
4141from searx.utils import (
4242 UnicodeWriter, highlight_content, html_to_text, get_themes
4343)
44+from searx.https_rewrite import https_rules
4445from searx.languages import language_codes
4546from searx.search import Search
4647from searx.autocomplete import backends as autocomplete_backends
4748
4849
49−static_path, templates_path, themes = get_themes(settings['themes_path'] if \
50− settings.get('themes_path', None) else searx_dir)
50+static_path, templates_path, themes =\
51+ get_themes(settings['themes_path']
52+ if settings.get('themes_path')
53+ else searx_dir)
5154default_theme = settings['default_theme'] if \
5255 settings.get('default_theme', None) else 'default'
5356
@@ -192,8 +195,20 @@ def index():
192195 search.lang)
193196
194197 for result in search.results:
198+
195199 if not search.paging and engines[result['engine']].paging:
196200 search.paging = True
201+
202+ if settings['server']['https_rewrite']\
203+ and result['parsed_url'].scheme == 'http':
204+
205+ for http_regex, https_url in https_rules:
206+ if http_regex.match(result['url']):
207+ result['url'] = http_regex.sub(https_url, result['url'])
208+ # TODO result['parsed_url'].scheme
209+ break
210+
211+ # HTTPS rewrite
197212 if search.request_data.get('format', 'html') == 'html':
198213 if 'content' in result:
199214 result['content'] = highlight_content(result['content'],
@@ -206,6 +221,7 @@ def index():
206221 # removing html content and whitespace duplications
207222 result['title'] = ' '.join(html_to_text(result['title'])
208223 .strip().split())
224+
209225 if len(result['url']) > 74:
210226 url_parts = result['url'][:35], result['url'][-35:]
211227 result['pretty_url'] = u'{0}[...]{1}'.format(*url_parts)