git.iko.soy/searx
on master help sign in
[mod] https rewrite pluginification
d2a636fAdam Tauber
searx/__init__.py−11
compare with master
unifiedsplit
@@ -36,11 +36,6 @@ if 'SEARX_SETTINGS_PATH' in environ:
36else:
36else:
37 settings_path = join(searx_dir, 'settings.yml')
37 settings_path = join(searx_dir, 'settings.yml')
38
38
39−if 'SEARX_HTTPS_REWRITE_PATH' in environ:
40− https_rewrite_path = environ['SEARX_HTTPS_REWRITE_PATH']
41−else:
42− https_rewrite_path = join(searx_dir, 'https_rules')
43−
44# load settings
39# load settings
45with open(settings_path) as settings_yaml:
40with open(settings_path) as settings_yaml:
46 settings = load(settings_yaml)
41 settings = load(settings_yaml)
@@ -52,10 +47,4 @@ else:
52
47
53logger = logging.getLogger('searx')
48logger = logging.getLogger('searx')
54
49
55−# load https rules only if https rewrite is enabled
56−if settings.get('server', {}).get('https_rewrite'):
57− # loade https rules
58− from searx.https_rewrite import load_https_rules
59− load_https_rules(https_rewrite_path)
60−
61logger.info('Initialisation done')
50logger.info('Initialisation done')
searx/plugins/__init__.py+6−3
@@ -14,13 +14,15 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
14
14
15(C) 2015 by Adam Tauber, <asciimoo@gmail.com>
15(C) 2015 by Adam Tauber, <asciimoo@gmail.com>
16'''
16'''
17−from searx.plugins import (self_ip,
18− search_on_category_select)
19−from searx import logger
20from sys import exit
17from sys import exit
18+from searx import logger
21
19
22logger = logger.getChild('plugins')
20logger = logger.getChild('plugins')
23
21
22+from searx.plugins import (https_rewrite,
23+ self_ip,
24+ search_on_category_select)
25+
24required_attrs = (('name', str),
26required_attrs = (('name', str),
25 ('description', str),
27 ('description', str),
26 ('default_on', bool))
28 ('default_on', bool))
@@ -68,5 +70,6 @@ class PluginStore():
68
70
69
71
70plugins = PluginStore()
72plugins = PluginStore()
73+plugins.register(https_rewrite)
71plugins.register(self_ip)
74plugins.register(self_ip)
72plugins.register(search_on_category_select)
75plugins.register(search_on_category_select)
searx/https_rewrite.py → searx/plugins/https_rewrite.pyrenamed+22−4
@@ -18,11 +18,22 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
18import re
18import re
19from urlparse import urlparse
19from urlparse import urlparse
20from lxml import etree
20from lxml import etree
21−from os import listdir
21+from os import listdir, environ
22from os.path import isfile, isdir, join
22from os.path import isfile, isdir, join
23−from searx import logger
23+from searx.plugins import logger
24+from flask.ext.babel import gettext
25+from searx import searx_dir
24
26
25
27
28+name = "HTTPS rewrite"
29+description = gettext('Rewrite HTTP links to HTTPS if possible')
30+default_on = True
31+
32+if 'SEARX_HTTPS_REWRITE_PATH' in environ:
33+ rules_path = environ['SEARX_rules_path']
34+else:
35+ rules_path = join(searx_dir, 'plugins/https_rules')
36+
26logger = logger.getChild("https_rewrite")
37logger = logger.getChild("https_rewrite")
27
38
28# https://gitweb.torproject.org/\
39# https://gitweb.torproject.org/\
@@ -33,7 +44,7 @@ https_rules = []
33
44
34
45
35# load single ruleset from a xml file
46# load single ruleset from a xml file
36−def load_single_https_ruleset(filepath):
47+def load_single_https_ruleset(rules_path):
37 ruleset = ()
48 ruleset = ()
38
49
39 # init parser
50 # init parser
@@ -41,7 +52,7 @@ def load_single_https_ruleset(filepath):
41
52
42 # load and parse xml-file
53 # load and parse xml-file
43 try:
54 try:
44− tree = etree.parse(filepath, parser)
55+ tree = etree.parse(rules_path, parser)
45 except:
56 except:
46 # TODO, error message
57 # TODO, error message
47 return ()
58 return ()
@@ -207,3 +218,10 @@ def https_url_rewrite(result):
207 # target has matched, do not search over the other rules
218 # target has matched, do not search over the other rules
208 break
219 break
209 return result
220 return result
221+
222+
223+def on_result(request, ctx):
224+ result = ctx['result']
225+ if result['parsed_url'].scheme == 'http':
226+ https_url_rewrite(result)
227+ return True
searx/https_rules/00README → searx/plugins/https_rules/00READMErenamed

The content is the same.

searx/https_rules/Bing.xml → searx/plugins/https_rules/Bing.xmlrenamed

The content is the same.

searx/https_rules/Dailymotion.xml → searx/plugins/https_rules/Dailymotion.xmlrenamed

The content is the same.

searx/https_rules/Deviantart.xml → searx/plugins/https_rules/Deviantart.xmlrenamed

The content is the same.

searx/https_rules/DuckDuckGo.xml → searx/plugins/https_rules/DuckDuckGo.xmlrenamed

The content is the same.

searx/https_rules/Flickr.xml → searx/plugins/https_rules/Flickr.xmlrenamed

The content is the same.

searx/https_rules/Github-Pages.xml → searx/plugins/https_rules/Github-Pages.xmlrenamed

The content is the same.

searx/https_rules/Github.xml → searx/plugins/https_rules/Github.xmlrenamed

The content is the same.

searx/https_rules/Google-mismatches.xml → searx/plugins/https_rules/Google-mismatches.xmlrenamed

The content is the same.

searx/https_rules/Google.org.xml → searx/plugins/https_rules/Google.org.xmlrenamed

The content is the same.

searx/https_rules/GoogleAPIs.xml → searx/plugins/https_rules/GoogleAPIs.xmlrenamed

The content is the same.

searx/https_rules/GoogleCanada.xml → searx/plugins/https_rules/GoogleCanada.xmlrenamed

The content is the same.

searx/https_rules/GoogleImages.xml → searx/plugins/https_rules/GoogleImages.xmlrenamed

The content is the same.

searx/https_rules/GoogleMainSearch.xml → searx/plugins/https_rules/GoogleMainSearch.xmlrenamed

The content is the same.

searx/https_rules/GoogleMaps.xml → searx/plugins/https_rules/GoogleMaps.xmlrenamed

The content is the same.

searx/https_rules/GoogleMelange.xml → searx/plugins/https_rules/GoogleMelange.xmlrenamed

The content is the same.

searx/https_rules/GoogleSearch.xml → searx/plugins/https_rules/GoogleSearch.xmlrenamed

The content is the same.

searx/https_rules/GoogleServices.xml → searx/plugins/https_rules/GoogleServices.xmlrenamed

The content is the same.

searx/https_rules/GoogleShopping.xml → searx/plugins/https_rules/GoogleShopping.xmlrenamed

The content is the same.

searx/https_rules/GoogleSorry.xml → searx/plugins/https_rules/GoogleSorry.xmlrenamed

The content is the same.

searx/https_rules/GoogleTranslate.xml → searx/plugins/https_rules/GoogleTranslate.xmlrenamed

The content is the same.

searx/https_rules/GoogleVideos.xml → searx/plugins/https_rules/GoogleVideos.xmlrenamed

The content is the same.

searx/https_rules/GoogleWatchBlog.xml → searx/plugins/https_rules/GoogleWatchBlog.xmlrenamed

The content is the same.

searx/https_rules/Google_App_Engine.xml → searx/plugins/https_rules/Google_App_Engine.xmlrenamed

The content is the same.

searx/https_rules/Googleplex.com.xml → searx/plugins/https_rules/Googleplex.com.xmlrenamed

The content is the same.

searx/https_rules/OpenStreetMap.xml → searx/plugins/https_rules/OpenStreetMap.xmlrenamed

The content is the same.

searx/https_rules/Rawgithub.com.xml → searx/plugins/https_rules/Rawgithub.com.xmlrenamed

The content is the same.

searx/https_rules/Soundcloud.xml → searx/plugins/https_rules/Soundcloud.xmlrenamed

The content is the same.

searx/https_rules/ThePirateBay.xml → searx/plugins/https_rules/ThePirateBay.xmlrenamed

The content is the same.

searx/https_rules/Torproject.xml → searx/plugins/https_rules/Torproject.xmlrenamed

The content is the same.

searx/https_rules/Twitter.xml → searx/plugins/https_rules/Twitter.xmlrenamed

The content is the same.

searx/https_rules/Vimeo.xml → searx/plugins/https_rules/Vimeo.xmlrenamed

The content is the same.

searx/https_rules/WikiLeaks.xml → searx/plugins/https_rules/WikiLeaks.xmlrenamed

The content is the same.

searx/https_rules/Wikimedia.xml → searx/plugins/https_rules/Wikimedia.xmlrenamed

The content is the same.

searx/https_rules/Yahoo.xml → searx/plugins/https_rules/Yahoo.xmlrenamed

The content is the same.

searx/https_rules/YouTube.xml → searx/plugins/https_rules/YouTube.xmlrenamed

The content is the same.

searx/settings.yml−1
@@ -6,7 +6,6 @@ server:
6 base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/"
6 base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/"
7 themes_path : "" # Custom ui themes path - leave it blank if you didn't change
7 themes_path : "" # Custom ui themes path - leave it blank if you didn't change
8 default_theme : oscar # ui theme
8 default_theme : oscar # ui theme
9− https_rewrite : True # Force rewrite result urls. See searx/https_rewrite.py
10 useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
9 useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
11 image_proxy : False # Proxying image results through searx
10 image_proxy : False # Proxying image results through searx
12 default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
11 default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
searx/webapp.py+1−7
@@ -59,7 +59,6 @@ from searx.utils import (
59)
59)
60from searx.version import VERSION_STRING
60from searx.version import VERSION_STRING
61from searx.languages import language_codes
61from searx.languages import language_codes
62−from searx.https_rewrite import https_url_rewrite
63from searx.search import Search
62from searx.search import Search
64from searx.query import Query
63from searx.query import Query
65from searx.autocomplete import searx_bang, backends as autocomplete_backends
64from searx.autocomplete import searx_bang, backends as autocomplete_backends
@@ -359,15 +358,10 @@ def index():
359
358
360 for result in search.results:
359 for result in search.results:
361
360
361+ plugins.call('on_result', request, locals())
362 if not search.paging and engines[result['engine']].paging:
362 if not search.paging and engines[result['engine']].paging:
363 search.paging = True
363 search.paging = True
364
364
365− # check if HTTPS rewrite is required
366− if settings['server']['https_rewrite']\
367− and result['parsed_url'].scheme == 'http':
368−
369− result = https_url_rewrite(result)
370−
371 if search.request_data.get('format', 'html') == 'html':
365 if search.request_data.get('format', 'html') == 'html':
372 if 'content' in result:
366 if 'content' in result:
373 result['content'] = highlight_content(result['content'],
367 result['content'] = highlight_content(result['content'],