bd92b43 · searx · git.iko.soy

searx/plugins/__init__.pynew file+48

@@ -0,0 +1,48 @@

1+from searx.plugins import self_ip

2+from searx import logger

3+from sys import exit

4+

5+logger = logger.getChild('plugins')

6+

7+required_attrs = (('name', str),

8+ ('description', str),

9+ ('default_on', bool))

10+

11+

12+class Plugin():

13+ default_on = False

14+ name = 'Default plugin'

15+ description = 'Default plugin description'

16+

17+

18+class PluginStore():

19+

20+ def __init__(self):

21+ self.plugins = []

22+

23+ def __iter__(self):

24+ for plugin in self.plugins:

25+ yield plugin

26+

27+ def register(self, *plugins):

28+ for plugin in plugins:

29+ for plugin_attr, plugin_attr_type in required_attrs:

30+ if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):

31+ logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))

32+ exit(3)

33+ plugin.id = plugin.name.replace(' ', '_')

34+ self.plugins.append(plugin)

35+

36+ def call(self, plugin_type, request, *args, **kwargs):

37+ ret = True

38+ for plugin in request.user_plugins:

39+ if hasattr(plugin, plugin_type):

40+ ret = getattr(plugin, plugin_type)(request, *args, **kwargs)

41+ if not ret:

42+ break

43+

44+ return ret

45+

46+

47+plugins = PluginStore()

48+plugins.register(self_ip)

searx/plugins/self_ip.pynew file+21

@@ -0,0 +1,21 @@

1+from flask.ext.babel import gettext

2+name = "Self IP"

3+description = gettext('Display your source IP address if the query expression is "ip"')

4+default_on = True

5+

6+

7+# attach callback to the pre search hook

8+# request: flask request object

9+# ctx: the whole local context of the pre search hook

10+def pre_search(request, ctx):

11+ if ctx['search'].query == 'ip':

12+ x_forwarded_for = request.headers.getlist("X-Forwarded-For")

13+ if x_forwarded_for:

14+ ip = x_forwarded_for[0]

15+ else:

16+ ip = request.remote_addr

17+ ctx['search'].answers.clear()

18+ ctx['search'].answers.add(ip)

19+ # return False prevents exeecution of the original block

20+ return False

21+ return True

searx/search.py+12−15

@@ -329,8 +329,8 @@ class Search(object):

329329 self.blocked_engines = get_blocked_engines(engines, request.cookies)

330330

331331 self.results = []

332− self.suggestions = []

333− self.answers = []

332+ self.suggestions = set()

333+ self.answers = set()

334334 self.infoboxes = []

335335 self.request_data = {}

336336

@@ -429,9 +429,6 @@ class Search(object):

429429 requests = []

430430 results_queue = Queue()

431431 results = {}

432− suggestions = set()

433− answers = set()

434− infoboxes = []

435432

436433 # increase number of searches

437434 number_of_searches += 1

@@ -511,7 +508,7 @@ class Search(object):

511508 selected_engine['name']))

512509

513510 if not requests:

514− return results, suggestions, answers, infoboxes

511+ return self

515512 # send all search-request

516513 threaded_requests(requests)

517514

@@ -519,19 +516,19 @@ class Search(object):

519516 engine_name, engine_results = results_queue.get_nowait()

520517

521518 # TODO type checks

522− [suggestions.add(x['suggestion'])

519+ [self.suggestions.add(x['suggestion'])

523520 for x in list(engine_results)

524521 if 'suggestion' in x

525522 and engine_results.remove(x) is None]

526523

527− [answers.add(x['answer'])

524+ [self.answers.add(x['answer'])

528525 for x in list(engine_results)

529526 if 'answer' in x

530527 and engine_results.remove(x) is None]

531528

532− infoboxes.extend(x for x in list(engine_results)

533− if 'infobox' in x

534− and engine_results.remove(x) is None)

529+ self.infoboxes.extend(x for x in list(engine_results)

530+ if 'infobox' in x

531+ and engine_results.remove(x) is None)

535532

536533 results[engine_name] = engine_results

537534

@@ -541,16 +538,16 @@ class Search(object):

541538 engines[engine_name].stats['result_count'] += len(engine_results)

542539

543540 # score results and remove duplications

544− results = score_results(results)

541+ self.results = score_results(results)

545542

546543 # merge infoboxes according to their ids

547− infoboxes = merge_infoboxes(infoboxes)

544+ self.infoboxes = merge_infoboxes(self.infoboxes)

548545

549546 # update engine stats, using calculated score

550− for result in results:

547+ for result in self.results:

551548 for res_engine in result['engines']:

552549 engines[result['engine']]\

553550 .stats['score_count'] += result['score']

554551

555552 # return results, suggestions, answers and infoboxes

556− return results, suggestions, answers, infoboxes

553+ return self

searx/settings.yml+1

@@ -106,6 +106,7 @@ engines:

106106 - name : gigablast

107107 engine : gigablast

108108 shortcut : gb

109+ disabled: True

109110

110111 - name : github

111112 engine : github

searx/templates/oscar/macros.html+8

@@ -59,3 +59,11 @@

5959 </div>

6060 {% endif %}

6161{%- endmacro %}

62+

63+{% macro checkbox_toggle(id, blocked) -%}

64+ <div class="checkbox">

65+ <input class="hidden" type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} />

66+ <label class="btn btn-success label_hide_if_checked" for="{{ id }}">{{ _('Block') }}</label>

67+ <label class="btn btn-danger label_hide_if_not_checked" for="{{ id }}">{{ _('Allow') }}</label>

68+ </div>

69+{%- endmacro %}

searx/templates/oscar/preferences.html+25−6

@@ -1,4 +1,4 @@

1−{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl %}

1+{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl, checkbox_toggle %}

22{% extends "oscar/base.html" %}

33{% block title %}{{ _('preferences') }} - {% endblock %}

44{% block site_alert_warning_nojs %}

@@ -16,6 +16,7 @@

1616 <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;">

1717 <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>

1818 <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>

19+ <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>

1920 </ul>

2021

2122

@@ -139,11 +140,7 @@

139140 <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div>

140141 {% endif %}

141142 <div class="col-xs-6 col-sm-4 col-md-4">

142− <div class="checkbox">

143− <input class="hidden" type="checkbox" id="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}" name="engine_{{ search_engine.name }}__{{ categ }}"{% if (search_engine.name, categ) in blocked_engines %} checked="checked"{% endif %} />

144− <label class="btn btn-success label_hide_if_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Block') }}</label>

145− <label class="btn btn-danger label_hide_if_not_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Allow') }}</label>

146− </div>

143+ {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }}

147144 </div>

148145 {% if rtl %}

149146 <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})&lrm;</div>

@@ -157,6 +154,28 @@

157154 {% endfor %}

158155 </div>

159156 </div>

157+ <div class="tab-pane active_if_nojs" id="tab_plugins">

158+ <noscript>

159+ <h3>{{ _('Plugins') }}</h3>

160+ </noscript>

161+ <fieldset>

162+ <div class="container-fluid">

163+ {% for plugin in plugins %}

164+ <div class="panel panel-default">

165+ <div class="panel-heading">

166+ <h3 class="panel-title">{{ plugin.name }}</h3>

167+ </div>

168+ <div class="panel-body">

169+ <div class="col-xs-6 col-sm-4 col-md-6">{{ plugin.description }}</div>

170+ <div class="col-xs-6 col-sm-4 col-md-6">

171+ {{ checkbox_toggle('plugin_' + plugin.id, plugin.id not in allowed_plugins) }}

172+ </div>

173+ </div>

174+ </div>

175+ {% endfor %}

176+ </div>

177+ </fieldset>

178+ </div>

160179 </div>

161180 <p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }}

162181 <br />

searx/templates/oscar/results.html+5−5

@@ -25,8 +25,8 @@

2525 {% endif %}

2626 </div>

2727 {% endfor %}

28−

29− {% if not results %}

28+

29+ {% if not results and not answers %}

3030 {% include 'oscar/messages/no_results.html' %}

3131 {% endif %}

3232

@@ -82,7 +82,7 @@

8282 {% for infobox in infoboxes %}

8383 {% include 'oscar/infobox.html' %}

8484 {% endfor %}

85− {% endif %}

85+ {% endif %}

8686

8787 {% if suggestions %}

8888 <div class="panel panel-default">

@@ -111,7 +111,7 @@

111111 <input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ base_url }}?q={{ q|urlencode }}&pageno={{ pageno }}{% if selected_categories %}&category_{{ selected_categories|join("&category_")|replace(' ','+') }}{% endif %}" readonly>

112112 </div>

113113 </form>

114−

114+

115115 <label>{{ _('Download results') }}</label>

116116 <div class="clearfix"></div>

117117 {% for output_type in ('csv', 'json', 'rss') %}

@@ -122,7 +122,7 @@

122122 <input type="hidden" name="pageno" value="{{ pageno }}">

123123 <button type="submit" class="btn btn-default">{{ output_type }}</button>

124124 </form>

125− {% endfor %}

125+ {% endfor %}

126126 <div class="clearfix"></div>

127127 </div>

128128 </div>

searx/tests/test_plugins.pynew file+51

@@ -0,0 +1,51 @@

1+# -*- coding: utf-8 -*-

2+

3+from searx.testing import SearxTestCase

4+from searx import plugins

5+from mock import Mock

6+

7+

8+class PluginStoreTest(SearxTestCase):

9+

10+ def test_PluginStore_init(self):

11+ store = plugins.PluginStore()

12+ self.assertTrue(isinstance(store.plugins, list) and len(store.plugins) == 0)

13+

14+ def test_PluginStore_register(self):

15+ store = plugins.PluginStore()

16+ testplugin = plugins.Plugin()

17+ store.register(testplugin)

18+

19+ self.assertTrue(len(store.plugins) == 1)

20+

21+ def test_PluginStore_call(self):

22+ store = plugins.PluginStore()

23+ testplugin = plugins.Plugin()

24+ store.register(testplugin)

25+ setattr(testplugin, 'asdf', Mock())

26+ request = Mock(user_plugins=[])

27+ store.call('asdf', request, Mock())

28+

29+ self.assertFalse(testplugin.asdf.called)

30+

31+ request.user_plugins.append(testplugin)

32+ store.call('asdf', request, Mock())

33+

34+ self.assertTrue(testplugin.asdf.called)

35+

36+

37+class SelfIPTest(SearxTestCase):

38+

39+ def test_PluginStore_init(self):

40+ store = plugins.PluginStore()

41+ store.register(plugins.self_ip)

42+

43+ self.assertTrue(len(store.plugins) == 1)

44+

45+ request = Mock(user_plugins=store.plugins,

46+ remote_addr='127.0.0.1')

47+ request.headers.getlist.return_value = []

48+ ctx = {'search': Mock(answers=set(),

49+ query='ip')}

50+ store.call('pre_search', request, ctx)

51+ self.assertTrue('127.0.0.1' in ctx['search'].answers)

searx/tests/test_webapp.py+9−33

@@ -2,7 +2,6 @@

22

33import json

44from urlparse import ParseResult

5−from mock import patch

65from searx import webapp

76from searx.testing import SearxTestCase

87

@@ -33,6 +32,11 @@ class ViewsTestCase(SearxTestCase):

3332 },

3433 ]

3534

35+ def search_mock(search_self, *args):

36+ search_self.results = self.test_results

37+

38+ webapp.Search.search = search_mock

39+

3640 self.maxDiff = None # to see full diffs

3741

3842 def test_index_empty(self):

@@ -40,14 +44,7 @@ class ViewsTestCase(SearxTestCase):

4044 self.assertEqual(result.status_code, 200)

4145 self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)

4246

43− @patch('searx.search.Search.search')

44− def test_index_html(self, search):

45− search.return_value = (

46− self.test_results,

47− set(),

48− set(),

49− set()

50− )

47+ def test_index_html(self):

5148 result = self.app.post('/', data={'q': 'test'})

5249 self.assertIn(

5350 '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa

@@ -58,14 +55,7 @@ class ViewsTestCase(SearxTestCase):

5855 result.data

5956 )

6057

61− @patch('searx.search.Search.search')

62− def test_index_json(self, search):

63− search.return_value = (

64− self.test_results,

65− set(),

66− set(),

67− set()

68− )

58+ def test_index_json(self):

6959 result = self.app.post('/', data={'q': 'test', 'format': 'json'})

7060

7161 result_dict = json.loads(result.data)

@@ -76,14 +66,7 @@ class ViewsTestCase(SearxTestCase):

7666 self.assertEqual(

7767 result_dict['results'][0]['url'], 'http://first.test.xyz')

7868

79− @patch('searx.search.Search.search')

80− def test_index_csv(self, search):

81− search.return_value = (

82− self.test_results,

83− set(),

84− set(),

85− set()

86− )

69+ def test_index_csv(self):

8770 result = self.app.post('/', data={'q': 'test', 'format': 'csv'})

8871

8972 self.assertEqual(

@@ -93,14 +76,7 @@ class ViewsTestCase(SearxTestCase):

9376 result.data

9477 )

9578

96− @patch('searx.search.Search.search')

97− def test_index_rss(self, search):

98− search.return_value = (

99− self.test_results,

100− set(),

101− set(),

102− set()

103− )

79+ def test_index_rss(self):

10480 result = self.app.post('/', data={'q': 'test', 'format': 'rss'})

10581

10682 self.assertIn(

searx/webapp.py+58−15

@@ -27,6 +27,18 @@ import cStringIO

2727import os

2828import hashlib

2929

30+from searx import logger

31+logger = logger.getChild('webapp')

32+

33+try:

34+ from pygments import highlight

35+ from pygments.lexers import get_lexer_by_name

36+ from pygments.formatters import HtmlFormatter

37+except:

38+ logger.critical("cannot import dependency: pygments")

39+ from sys import exit

40+ exit(1)

41+

3042from datetime import datetime, timedelta

3143from urllib import urlencode

3244from werkzeug.contrib.fixers import ProxyFix

@@ -51,18 +63,8 @@ from searx.https_rewrite import https_url_rewrite

5163from searx.search import Search

5264from searx.query import Query

5365from searx.autocomplete import searx_bang, backends as autocomplete_backends

54−from searx import logger

55−try:

56− from pygments import highlight

57− from pygments.lexers import get_lexer_by_name

58− from pygments.formatters import HtmlFormatter

59−except:

60− logger.critical("cannot import dependency: pygments")

61− from sys import exit

62− exit(1)

63−

66+from searx.plugins import plugins

6467

65−logger = logger.getChild('webapp')

6668

6769static_path, templates_path, themes =\

6870 get_themes(settings['themes_path']

@@ -303,6 +305,23 @@ def render(template_name, override_theme=None, **kwargs):

303305 '{}/{}'.format(kwargs['theme'], template_name), **kwargs)

304306

305307

308+@app.before_request

309+def pre_request():

310+ # merge GET, POST vars

311+ request.form = dict(request.form.items())

312+ for k, v in request.args:

313+ if k not in request.form:

314+ request.form[k] = v

315+

316+ request.user_plugins = []

317+ allowed_plugins = request.cookies.get('allowed_plugins', '').split(',')

318+ disabled_plugins = request.cookies.get('disabled_plugins', '').split(',')

319+ for plugin in plugins:

320+ if ((plugin.default_on and plugin.id not in disabled_plugins)

321+ or plugin.id in allowed_plugins):

322+ request.user_plugins.append(plugin)

323+

324+

306325@app.route('/search', methods=['GET', 'POST'])

307326@app.route('/', methods=['GET', 'POST'])

308327def index():

@@ -323,8 +342,10 @@ def index():

323342 'index.html',

324343 )

325344

326− search.results, search.suggestions,\

327− search.answers, search.infoboxes = search.search(request)

345+ if plugins.call('pre_search', request, locals()):

346+ search.search(request)

347+

348+ plugins.call('post_search', request, locals())

328349

329350 for result in search.results:

330351

@@ -487,11 +508,11 @@ def preferences():

487508 blocked_engines = get_blocked_engines(engines, request.cookies)

488509 else: # on save

489510 selected_categories = []

511+ post_disabled_plugins = []

490512 locale = None

491513 autocomplete = ''

492514 method = 'POST'

493515 safesearch = '1'

494−

495516 for pd_name, pd in request.form.items():

496517 if pd_name.startswith('category_'):

497518 category = pd_name[9:]

@@ -514,14 +535,34 @@ def preferences():

514535 safesearch = pd

515536 elif pd_name.startswith('engine_'):

516537 if pd_name.find('__') > -1:

517− engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1)

538+ # TODO fix underscore vs space

539+ engine_name, category = [x.replace('_', ' ') for x in

540+ pd_name.replace('engine_', '', 1).split('__', 1)]

518541 if engine_name in engines and category in engines[engine_name].categories:

519542 blocked_engines.append((engine_name, category))

520543 elif pd_name == 'theme':

521544 theme = pd if pd in themes else default_theme

545+ elif pd_name.startswith('plugin_'):

546+ plugin_id = pd_name.replace('plugin_', '', 1)

547+ if not any(plugin.id == plugin_id for plugin in plugins):

548+ continue

549+ post_disabled_plugins.append(plugin_id)

522550 else:

523551 resp.set_cookie(pd_name, pd, max_age=cookie_max_age)

524552

553+ disabled_plugins = []

554+ allowed_plugins = []

555+ for plugin in plugins:

556+ if plugin.default_on:

557+ if plugin.id in post_disabled_plugins:

558+ disabled_plugins.append(plugin.id)

559+ elif plugin.id not in post_disabled_plugins:

560+ allowed_plugins.append(plugin.id)

561+

562+ resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age)

563+

564+ resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age)

565+

525566 resp.set_cookie(

526567 'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),

527568 max_age=cookie_max_age

@@ -571,6 +612,8 @@ def preferences():

571612 autocomplete_backends=autocomplete_backends,

572613 shortcuts={y: x for x, y in engine_shortcuts.items()},

573614 themes=themes,

615+ plugins=plugins,

616+ allowed_plugins=[plugin.id for plugin in request.user_plugins],

574617 theme=get_current_theme_name())

575618

576619