d2a636f · searx · git.iko.soy

git.iko.soy/searx

[mod] https rewrite pluginification

commitd2a636f75d24953f5094ea97ab54a8a4353a65ff authorAdam Tauber <asciimoo@gmail.com> dateMon 13 Apr 2015, 00:30 (12 years ago) parent146928a

41 files changed+29−26

compare with master

unifiedsplit

searx/__init__.py−11

@@ -36,11 +36,6 @@ if 'SEARX_SETTINGS_PATH' in environ:

3636else:

3737 settings_path = join(searx_dir, 'settings.yml')

3838

39−if 'SEARX_HTTPS_REWRITE_PATH' in environ:

40− https_rewrite_path = environ['SEARX_HTTPS_REWRITE_PATH']

41−else:

42− https_rewrite_path = join(searx_dir, 'https_rules')

43−

4439# load settings

4540with open(settings_path) as settings_yaml:

4641 settings = load(settings_yaml)

@@ -52,10 +47,4 @@ else:

5247

5348logger = logging.getLogger('searx')

5449

55−# load https rules only if https rewrite is enabled

56−if settings.get('server', {}).get('https_rewrite'):

57− # loade https rules

58− from searx.https_rewrite import load_https_rules

59− load_https_rules(https_rewrite_path)

60−

6150logger.info('Initialisation done')

searx/plugins/__init__.py+6−3

@@ -14,13 +14,15 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1414

1616'''

17−from searx.plugins import (self_ip,

18− search_on_category_select)

19−from searx import logger

2017from sys import exit

18+from searx import logger

2119

2220logger = logger.getChild('plugins')

2321

22+from searx.plugins import (https_rewrite,

23+ self_ip,

24+ search_on_category_select)

25+

2426required_attrs = (('name', str),

2527 ('description', str),

2628 ('default_on', bool))

@@ -68,5 +70,6 @@ class PluginStore():

6870

6971

7072plugins = PluginStore()

73+plugins.register(https_rewrite)

7174plugins.register(self_ip)

7275plugins.register(search_on_category_select)

searx/https_rewrite.py → searx/plugins/https_rewrite.pyrenamed+22−4

@@ -18,11 +18,22 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1818import re

1919from urlparse import urlparse

2020from lxml import etree

21−from os import listdir

21+from os import listdir, environ

2222from os.path import isfile, isdir, join

23−from searx import logger

23+from searx.plugins import logger

24+from flask.ext.babel import gettext

25+from searx import searx_dir

2426

2527

28+name = "HTTPS rewrite"

29+description = gettext('Rewrite HTTP links to HTTPS if possible')

30+default_on = True

31+

32+if 'SEARX_HTTPS_REWRITE_PATH' in environ:

33+ rules_path = environ['SEARX_rules_path']

34+else:

35+ rules_path = join(searx_dir, 'plugins/https_rules')

36+

2637logger = logger.getChild("https_rewrite")

2738

2839# https://gitweb.torproject.org/\

@@ -33,7 +44,7 @@ https_rules = []

3344

3445

3546# load single ruleset from a xml file

36−def load_single_https_ruleset(filepath):

47+def load_single_https_ruleset(rules_path):

3748 ruleset = ()

3849

3950 # init parser

@@ -41,7 +52,7 @@ def load_single_https_ruleset(filepath):

4152

4253 # load and parse xml-file

4354 try:

44− tree = etree.parse(filepath, parser)

55+ tree = etree.parse(rules_path, parser)

4556 except:

4657 # TODO, error message

4758 return ()

@@ -207,3 +218,10 @@ def https_url_rewrite(result):

207218 # target has matched, do not search over the other rules

208219 break

209220 return result

221+

222+

223+def on_result(request, ctx):

224+ result = ctx['result']

225+ if result['parsed_url'].scheme == 'http':

226+ https_url_rewrite(result)

227+ return True

searx/https_rules/00README → searx/plugins/https_rules/00READMErenamed

The content is the same.

searx/https_rules/Bing.xml → searx/plugins/https_rules/Bing.xmlrenamed

The content is the same.

searx/https_rules/Dailymotion.xml → searx/plugins/https_rules/Dailymotion.xmlrenamed

The content is the same.

searx/https_rules/Deviantart.xml → searx/plugins/https_rules/Deviantart.xmlrenamed

The content is the same.

searx/https_rules/DuckDuckGo.xml → searx/plugins/https_rules/DuckDuckGo.xmlrenamed

The content is the same.

searx/https_rules/Flickr.xml → searx/plugins/https_rules/Flickr.xmlrenamed

The content is the same.

searx/https_rules/Github-Pages.xml → searx/plugins/https_rules/Github-Pages.xmlrenamed

The content is the same.

searx/https_rules/Github.xml → searx/plugins/https_rules/Github.xmlrenamed

The content is the same.

searx/https_rules/Google-mismatches.xml → searx/plugins/https_rules/Google-mismatches.xmlrenamed

The content is the same.

searx/https_rules/Google.org.xml → searx/plugins/https_rules/Google.org.xmlrenamed

The content is the same.

searx/https_rules/GoogleAPIs.xml → searx/plugins/https_rules/GoogleAPIs.xmlrenamed

The content is the same.

searx/https_rules/GoogleCanada.xml → searx/plugins/https_rules/GoogleCanada.xmlrenamed

The content is the same.

searx/https_rules/GoogleImages.xml → searx/plugins/https_rules/GoogleImages.xmlrenamed

The content is the same.

searx/https_rules/GoogleMainSearch.xml → searx/plugins/https_rules/GoogleMainSearch.xmlrenamed

The content is the same.

searx/https_rules/GoogleMaps.xml → searx/plugins/https_rules/GoogleMaps.xmlrenamed

The content is the same.

searx/https_rules/GoogleMelange.xml → searx/plugins/https_rules/GoogleMelange.xmlrenamed

The content is the same.

searx/https_rules/GoogleSearch.xml → searx/plugins/https_rules/GoogleSearch.xmlrenamed

The content is the same.

searx/https_rules/GoogleServices.xml → searx/plugins/https_rules/GoogleServices.xmlrenamed

The content is the same.

searx/https_rules/GoogleShopping.xml → searx/plugins/https_rules/GoogleShopping.xmlrenamed

The content is the same.

searx/https_rules/GoogleSorry.xml → searx/plugins/https_rules/GoogleSorry.xmlrenamed

The content is the same.

searx/https_rules/GoogleTranslate.xml → searx/plugins/https_rules/GoogleTranslate.xmlrenamed

The content is the same.

searx/https_rules/GoogleVideos.xml → searx/plugins/https_rules/GoogleVideos.xmlrenamed

The content is the same.

searx/https_rules/GoogleWatchBlog.xml → searx/plugins/https_rules/GoogleWatchBlog.xmlrenamed

The content is the same.

searx/https_rules/Google_App_Engine.xml → searx/plugins/https_rules/Google_App_Engine.xmlrenamed

The content is the same.

searx/https_rules/Googleplex.com.xml → searx/plugins/https_rules/Googleplex.com.xmlrenamed

The content is the same.

searx/https_rules/OpenStreetMap.xml → searx/plugins/https_rules/OpenStreetMap.xmlrenamed

The content is the same.

searx/https_rules/Rawgithub.com.xml → searx/plugins/https_rules/Rawgithub.com.xmlrenamed

The content is the same.

searx/https_rules/Soundcloud.xml → searx/plugins/https_rules/Soundcloud.xmlrenamed

The content is the same.

searx/https_rules/ThePirateBay.xml → searx/plugins/https_rules/ThePirateBay.xmlrenamed

The content is the same.

searx/https_rules/Torproject.xml → searx/plugins/https_rules/Torproject.xmlrenamed

The content is the same.

searx/https_rules/Twitter.xml → searx/plugins/https_rules/Twitter.xmlrenamed

The content is the same.

searx/https_rules/Vimeo.xml → searx/plugins/https_rules/Vimeo.xmlrenamed

The content is the same.

searx/https_rules/WikiLeaks.xml → searx/plugins/https_rules/WikiLeaks.xmlrenamed

The content is the same.

searx/https_rules/Wikimedia.xml → searx/plugins/https_rules/Wikimedia.xmlrenamed

The content is the same.

searx/https_rules/Yahoo.xml → searx/plugins/https_rules/Yahoo.xmlrenamed

The content is the same.

searx/https_rules/YouTube.xml → searx/plugins/https_rules/YouTube.xmlrenamed

The content is the same.

searx/settings.yml−1

@@ -6,7 +6,6 @@ server:

66 base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/"

77 themes_path : "" # Custom ui themes path - leave it blank if you didn't change

88 default_theme : oscar # ui theme

9− https_rewrite : True # Force rewrite result urls. See searx/https_rewrite.py

109 useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator

1110 image_proxy : False # Proxying image results through searx

1211 default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section

searx/webapp.py+1−7

@@ -59,7 +59,6 @@ from searx.utils import (

5959)

6060from searx.version import VERSION_STRING

6161from searx.languages import language_codes

62−from searx.https_rewrite import https_url_rewrite

6362from searx.search import Search

6463from searx.query import Query

6564from searx.autocomplete import searx_bang, backends as autocomplete_backends

@@ -359,15 +358,10 @@ def index():

359358

360359 for result in search.results:

361360

361+ plugins.call('on_result', request, locals())

362362 if not search.paging and engines[result['engine']].paging:

363363 search.paging = True

364364

365− # check if HTTPS rewrite is required

366− if settings['server']['https_rewrite']\

367− and result['parsed_url'].scheme == 'http':

368−

369− result = https_url_rewrite(result)

370−

371365 if search.request_data.get('format', 'html') == 'html':

372366 if 'content' in result:

373367 result['content'] = highlight_content(result['content'],