bd92b43 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

Merge branch 'plugins'

bd92b43Adam Tauber12 years ago

searx/plugins/__init__.pynew file+48

compare with master

unifiedsplit

@@ -0,0 +1,48 @@

1+from searx.plugins import self_ip

2+from searx import logger

3+from sys import exit

5+logger = logger.getChild('plugins')

7+required_attrs = (('name', str),

8+ ('description', str),

9+ ('default_on', bool))

10+

11+

12+class Plugin():

13+ default_on = False

14+ name = 'Default plugin'

15+ description = 'Default plugin description'

16+

17+

18+class PluginStore():

19+

20+ def __init__(self):

21+ self.plugins = []

22+

23+ def __iter__(self):

24+ for plugin in self.plugins:

25+ yield plugin

26+

27+ def register(self, *plugins):

28+ for plugin in plugins:

29+ for plugin_attr, plugin_attr_type in required_attrs:

30+ if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):

31+ logger.critical('missing attribute "{0}", cannot load plugin: {1}'.format(plugin_attr, plugin))

32+ exit(3)

33+ plugin.id = plugin.name.replace(' ', '_')

34+ self.plugins.append(plugin)

35+

36+ def call(self, plugin_type, request, *args, **kwargs):

37+ ret = True

38+ for plugin in request.user_plugins:

39+ if hasattr(plugin, plugin_type):

40+ ret = getattr(plugin, plugin_type)(request, *args, **kwargs)

41+ if not ret:

42+ break

43+

44+ return ret

45+

46+

47+plugins = PluginStore()

48+plugins.register(self_ip)

searx/plugins/self_ip.pynew file+21

@@ -0,0 +1,21 @@

1+from flask.ext.babel import gettext

2+name = "Self IP"

3+description = gettext('Display your source IP address if the query expression is "ip"')

4+default_on = True

7+# attach callback to the pre search hook

8+# request: flask request object

9+# ctx: the whole local context of the pre search hook

10+def pre_search(request, ctx):

11+ if ctx['search'].query == 'ip':

12+ x_forwarded_for = request.headers.getlist("X-Forwarded-For")

13+ if x_forwarded_for:

14+ ip = x_forwarded_for[0]

15+ else:

16+ ip = request.remote_addr

17+ ctx['search'].answers.clear()

18+ ctx['search'].answers.add(ip)

19+ # return False prevents exeecution of the original block

20+ return False

21+ return True

searx/search.py+12−15

@@ -329,8 +329,8 @@ class Search(object):

329 self.blocked_engines = get_blocked_engines(engines, request.cookies)

330

331 self.results = []

332− self.suggestions = []

332+ self.suggestions = set()

333− self.answers = []

333+ self.answers = set()

334 self.infoboxes = []

335 self.request_data = {}

336

@@ -429,9 +429,6 @@ class Search(object):

429 requests = []

430 results_queue = Queue()

431 results = {}

432− suggestions = set()

433− answers = set()

434− infoboxes = []

435

432

436 # increase number of searches

433 # increase number of searches

437 number_of_searches += 1

434 number_of_searches += 1

@@ -511,7 +508,7 @@ class Search(object):

511 selected_engine['name']))

508 selected_engine['name']))

512

509

513 if not requests:

510 if not requests:

514− return results, suggestions, answers, infoboxes

511+ return self

515 # send all search-request

512 # send all search-request

516 threaded_requests(requests)

513 threaded_requests(requests)

517

514

@@ -519,19 +516,19 @@ class Search(object):

519 engine_name, engine_results = results_queue.get_nowait()

516 engine_name, engine_results = results_queue.get_nowait()

520

517

521 # TODO type checks

518 # TODO type checks

522− [suggestions.add(x['suggestion'])

519+ [self.suggestions.add(x['suggestion'])

523 for x in list(engine_results)

520 for x in list(engine_results)

524 if 'suggestion' in x

521 if 'suggestion' in x

525 and engine_results.remove(x) is None]

522 and engine_results.remove(x) is None]

526

523

527− [answers.add(x['answer'])

524+ [self.answers.add(x['answer'])

528 for x in list(engine_results)

525 for x in list(engine_results)

529 if 'answer' in x

526 if 'answer' in x

530 and engine_results.remove(x) is None]

527 and engine_results.remove(x) is None]

531

528

532− infoboxes.extend(x for x in list(engine_results)

529+ self.infoboxes.extend(x for x in list(engine_results)

533− if 'infobox' in x

530+ if 'infobox' in x

534− and engine_results.remove(x) is None)

531+ and engine_results.remove(x) is None)

535

532

536 results[engine_name] = engine_results

533 results[engine_name] = engine_results

537

534

@@ -541,16 +538,16 @@ class Search(object):

541 engines[engine_name].stats['result_count'] += len(engine_results)

538 engines[engine_name].stats['result_count'] += len(engine_results)

542

539

543 # score results and remove duplications

540 # score results and remove duplications

544− results = score_results(results)

541+ self.results = score_results(results)

545

542

546 # merge infoboxes according to their ids

543 # merge infoboxes according to their ids

547− infoboxes = merge_infoboxes(infoboxes)

544+ self.infoboxes = merge_infoboxes(self.infoboxes)

548

545

549 # update engine stats, using calculated score

546 # update engine stats, using calculated score

550− for result in results:

547+ for result in self.results:

551 for res_engine in result['engines']:

548 for res_engine in result['engines']:

552 engines[result['engine']]\

549 engines[result['engine']]\

553 .stats['score_count'] += result['score']

550 .stats['score_count'] += result['score']

554

551

555 # return results, suggestions, answers and infoboxes

552 # return results, suggestions, answers and infoboxes

556− return results, suggestions, answers, infoboxes

553+ return self

searx/settings.yml+1

@@ -106,6 +106,7 @@ engines:

106 - name : gigablast

107 engine : gigablast

108 shortcut : gb

109+ disabled: True

109

110

110 - name : github

111 - name : github

111 engine : github

112 engine : github

searx/templates/oscar/macros.html+8

@@ -59,3 +59,11 @@

59 </div>

60 {% endif %}

61{%- endmacro %}

62+

63+{% macro checkbox_toggle(id, blocked) -%}

64+ <div class="checkbox">

65+ <input class="hidden" type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} />

66+ <label class="btn btn-success label_hide_if_checked" for="{{ id }}">{{ _('Block') }}</label>

67+ <label class="btn btn-danger label_hide_if_not_checked" for="{{ id }}">{{ _('Allow') }}</label>

68+ </div>

69+{%- endmacro %}

searx/templates/oscar/preferences.html+25−6

@@ -1,4 +1,4 @@

1−{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl %}

1+{% from 'oscar/macros.html' import preferences_item_header, preferences_item_header_rtl, preferences_item_footer, preferences_item_footer_rtl, checkbox_toggle %}

2{% extends "oscar/base.html" %}

3{% block title %}{{ _('preferences') }} - {% endblock %}

4{% block site_alert_warning_nojs %}

@@ -16,6 +16,7 @@

16 <ul class="nav nav-tabs nav-justified hide_if_nojs" role="tablist" style="margin-bottom:20px;">

17 <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>

18 <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>

19+ <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>

19 </ul>

20 </ul>

21

22

@@ -139,11 +140,7 @@

139 <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div>

140 <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})</div>

140 {% endif %}

141 {% endif %}

141 <div class="col-xs-6 col-sm-4 col-md-4">

142 <div class="col-xs-6 col-sm-4 col-md-4">

142− <div class="checkbox">

143− <input class="hidden" type="checkbox" id="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}" name="engine_{{ search_engine.name }}__{{ categ }}"{% if (search_engine.name, categ) in blocked_engines %} checked="checked"{% endif %} />

143+ {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in blocked_engines) }}

144− <label class="btn btn-success label_hide_if_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Block') }}</label>

145− <label class="btn btn-danger label_hide_if_not_checked" for="engine_{{ categ|replace(' ', '_') }}_{{ search_engine.name|replace(' ', '_') }}">{{ _('Allow') }}</label>

146− </div>

147 </div>

144 </div>

148 {% if rtl %}

145 {% if rtl %}

149 <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})&lrm;</div>

146 <div class="col-xs-6 col-sm-4 col-md-4">{{ search_engine.name }} ({{ shortcuts[search_engine.name] }})&lrm;</div>

@@ -157,6 +154,28 @@

157 {% endfor %}

154 {% endfor %}

158 </div>

155 </div>

159 </div>

156 </div>

157+ <div class="tab-pane active_if_nojs" id="tab_plugins">

158+ <noscript>

159+ <h3>{{ _('Plugins') }}</h3>

160+ </noscript>

161+ <fieldset>

162+ <div class="container-fluid">

163+ {% for plugin in plugins %}

164+ <div class="panel panel-default">

165+ <div class="panel-heading">

166+ <h3 class="panel-title">{{ plugin.name }}</h3>

167+ </div>

168+ <div class="panel-body">

169+ <div class="col-xs-6 col-sm-4 col-md-6">{{ plugin.description }}</div>

170+ <div class="col-xs-6 col-sm-4 col-md-6">

171+ {{ checkbox_toggle('plugin_' + plugin.id, plugin.id not in allowed_plugins) }}

172+ </div>

173+ </div>

174+ </div>

175+ {% endfor %}

176+ </div>

177+ </fieldset>

178+ </div>

160 </div>

179 </div>

161 <p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }}

180 <p class="text-muted" style="margin:20px 0;">{{ _('These settings are stored in your cookies, this allows us not to store this data about you.') }}

162 <br />

181 <br />

searx/templates/oscar/results.html+5−5

@@ -25,8 +25,8 @@

25 {% endif %}

26 </div>

27 {% endfor %}

28−

28+

29− {% if not results %}

29+ {% if not results and not answers %}

30 {% include 'oscar/messages/no_results.html' %}

31 {% endif %}

@@ -82,7 +82,7 @@

82 {% for infobox in infoboxes %}

83 {% include 'oscar/infobox.html' %}

84 {% endfor %}

85− {% endif %}

85+ {% endif %}

87 {% if suggestions %}

88 <div class="panel panel-default">

@@ -111,7 +111,7 @@

111 <input id="search_url" type="url" class="form-control select-all-on-click cursor-text" name="search_url" value="{{ base_url }}?q={{ q|urlencode }}&pageno={{ pageno }}{% if selected_categories %}&category_{{ selected_categories|join("&category_")|replace(' ','+') }}{% endif %}" readonly>

112 </div>

113 </form>

114−

114+

115 <label>{{ _('Download results') }}</label>

116 <div class="clearfix"></div>

117 {% for output_type in ('csv', 'json', 'rss') %}

@@ -122,7 +122,7 @@

122 <input type="hidden" name="pageno" value="{{ pageno }}">

123 <button type="submit" class="btn btn-default">{{ output_type }}</button>

124 </form>

125− {% endfor %}

125+ {% endfor %}

126 <div class="clearfix"></div>

127 </div>

128 </div>

searx/tests/test_plugins.pynew file+51

@@ -0,0 +1,51 @@

1+# -*- coding: utf-8 -*-

3+from searx.testing import SearxTestCase

4+from searx import plugins

5+from mock import Mock

8+class PluginStoreTest(SearxTestCase):

10+ def test_PluginStore_init(self):

11+ store = plugins.PluginStore()

12+ self.assertTrue(isinstance(store.plugins, list) and len(store.plugins) == 0)

13+

14+ def test_PluginStore_register(self):

15+ store = plugins.PluginStore()

16+ testplugin = plugins.Plugin()

17+ store.register(testplugin)

18+

19+ self.assertTrue(len(store.plugins) == 1)

20+

21+ def test_PluginStore_call(self):

22+ store = plugins.PluginStore()

23+ testplugin = plugins.Plugin()

24+ store.register(testplugin)

25+ setattr(testplugin, 'asdf', Mock())

26+ request = Mock(user_plugins=[])

27+ store.call('asdf', request, Mock())

28+

29+ self.assertFalse(testplugin.asdf.called)

30+

31+ request.user_plugins.append(testplugin)

32+ store.call('asdf', request, Mock())

33+

34+ self.assertTrue(testplugin.asdf.called)

35+

36+

37+class SelfIPTest(SearxTestCase):

38+

39+ def test_PluginStore_init(self):

40+ store = plugins.PluginStore()

41+ store.register(plugins.self_ip)

42+

43+ self.assertTrue(len(store.plugins) == 1)

44+

45+ request = Mock(user_plugins=store.plugins,

46+ remote_addr='127.0.0.1')

47+ request.headers.getlist.return_value = []

48+ ctx = {'search': Mock(answers=set(),

49+ query='ip')}

50+ store.call('pre_search', request, ctx)

51+ self.assertTrue('127.0.0.1' in ctx['search'].answers)

searx/tests/test_webapp.py+9−33

@@ -2,7 +2,6 @@

3import json

4from urlparse import ParseResult

5−from mock import patch

6from searx import webapp

5from searx import webapp

7from searx.testing import SearxTestCase

6from searx.testing import SearxTestCase

@@ -33,6 +32,11 @@ class ViewsTestCase(SearxTestCase):

33 },

32 },

34 ]

33 ]

35+ def search_mock(search_self, *args):

36+ search_self.results = self.test_results

37+

38+ webapp.Search.search = search_mock

39+

36 self.maxDiff = None # to see full diffs

40 self.maxDiff = None # to see full diffs

38 def test_index_empty(self):

42 def test_index_empty(self):

@@ -40,14 +44,7 @@ class ViewsTestCase(SearxTestCase):

40 self.assertEqual(result.status_code, 200)

44 self.assertEqual(result.status_code, 200)

41 self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)

45 self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)

43− @patch('searx.search.Search.search')

44− def test_index_html(self, search):

47+ def test_index_html(self):

45− search.return_value = (

46− self.test_results,

47− set(),

48− set(),

49− set()

50− )

51 result = self.app.post('/', data={'q': 'test'})

48 result = self.app.post('/', data={'q': 'test'})

52 self.assertIn(

49 self.assertIn(

53 '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa

50 '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/default/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz">Second <span class="highlight">Test</span></a></h3>', # noqa

@@ -58,14 +55,7 @@ class ViewsTestCase(SearxTestCase):

58 result.data

55 result.data

59 )

56 )

61− @patch('searx.search.Search.search')

62− def test_index_json(self, search):

58+ def test_index_json(self):

63− search.return_value = (

64− self.test_results,

65− set(),

66− set(),

67− set()

68− )

69 result = self.app.post('/', data={'q': 'test', 'format': 'json'})

59 result = self.app.post('/', data={'q': 'test', 'format': 'json'})

71 result_dict = json.loads(result.data)

61 result_dict = json.loads(result.data)

@@ -76,14 +66,7 @@ class ViewsTestCase(SearxTestCase):

76 self.assertEqual(

66 self.assertEqual(

77 result_dict['results'][0]['url'], 'http://first.test.xyz')

67 result_dict['results'][0]['url'], 'http://first.test.xyz')

79− @patch('searx.search.Search.search')

80− def test_index_csv(self, search):

69+ def test_index_csv(self):

81− search.return_value = (

82− self.test_results,

83− set(),

84− set(),

85− set()

86− )

87 result = self.app.post('/', data={'q': 'test', 'format': 'csv'})

70 result = self.app.post('/', data={'q': 'test', 'format': 'csv'})

89 self.assertEqual(

72 self.assertEqual(

@@ -93,14 +76,7 @@ class ViewsTestCase(SearxTestCase):

93 result.data

76 result.data

94 )

77 )

96− @patch('searx.search.Search.search')

97− def test_index_rss(self, search):

79+ def test_index_rss(self):

98− search.return_value = (

99− self.test_results,

100− set(),

101− set(),

102− set()

103− )

104 result = self.app.post('/', data={'q': 'test', 'format': 'rss'})

80 result = self.app.post('/', data={'q': 'test', 'format': 'rss'})

105

106 self.assertIn(

82 self.assertIn(

searx/webapp.py+58−15

@@ -27,6 +27,18 @@ import cStringIO

27import os

28import hashlib

30+from searx import logger

31+logger = logger.getChild('webapp')

32+

33+try:

34+ from pygments import highlight

35+ from pygments.lexers import get_lexer_by_name

36+ from pygments.formatters import HtmlFormatter

37+except:

38+ logger.critical("cannot import dependency: pygments")

39+ from sys import exit

40+ exit(1)

41+

30from datetime import datetime, timedelta

42from datetime import datetime, timedelta

31from urllib import urlencode

43from urllib import urlencode

32from werkzeug.contrib.fixers import ProxyFix

44from werkzeug.contrib.fixers import ProxyFix

@@ -51,18 +63,8 @@ from searx.https_rewrite import https_url_rewrite

51from searx.search import Search

63from searx.search import Search

52from searx.query import Query

64from searx.query import Query

53from searx.autocomplete import searx_bang, backends as autocomplete_backends

65from searx.autocomplete import searx_bang, backends as autocomplete_backends

54−from searx import logger

66+from searx.plugins import plugins

55−try:

56− from pygments import highlight

57− from pygments.lexers import get_lexer_by_name

58− from pygments.formatters import HtmlFormatter

59−except:

60− logger.critical("cannot import dependency: pygments")

61− from sys import exit

62− exit(1)

63−

65−logger = logger.getChild('webapp')

67static_path, templates_path, themes =\

69static_path, templates_path, themes =\

68 get_themes(settings['themes_path']

70 get_themes(settings['themes_path']

@@ -303,6 +305,23 @@ def render(template_name, override_theme=None, **kwargs):

303 '{}/{}'.format(kwargs['theme'], template_name), **kwargs)

305 '{}/{}'.format(kwargs['theme'], template_name), **kwargs)

304

306

305

307

308+@app.before_request

309+def pre_request():

310+ # merge GET, POST vars

311+ request.form = dict(request.form.items())

312+ for k, v in request.args:

313+ if k not in request.form:

314+ request.form[k] = v

315+

316+ request.user_plugins = []

317+ allowed_plugins = request.cookies.get('allowed_plugins', '').split(',')

318+ disabled_plugins = request.cookies.get('disabled_plugins', '').split(',')

319+ for plugin in plugins:

320+ if ((plugin.default_on and plugin.id not in disabled_plugins)

321+ or plugin.id in allowed_plugins):

322+ request.user_plugins.append(plugin)

323+

324+

306@app.route('/search', methods=['GET', 'POST'])

325@app.route('/search', methods=['GET', 'POST'])

307@app.route('/', methods=['GET', 'POST'])

326@app.route('/', methods=['GET', 'POST'])

308def index():

327def index():

@@ -323,8 +342,10 @@ def index():

323 'index.html',

342 'index.html',

324 )

343 )

325

344

326− search.results, search.suggestions,\

345+ if plugins.call('pre_search', request, locals()):

327− search.answers, search.infoboxes = search.search(request)

346+ search.search(request)

347+

348+ plugins.call('post_search', request, locals())

328

349

329 for result in search.results:

350 for result in search.results:

330

351

@@ -487,11 +508,11 @@ def preferences():

487 blocked_engines = get_blocked_engines(engines, request.cookies)

508 blocked_engines = get_blocked_engines(engines, request.cookies)

488 else: # on save

509 else: # on save

489 selected_categories = []

510 selected_categories = []

511+ post_disabled_plugins = []

490 locale = None

512 locale = None

491 autocomplete = ''

513 autocomplete = ''

492 method = 'POST'

514 method = 'POST'

493 safesearch = '1'

515 safesearch = '1'

494−

495 for pd_name, pd in request.form.items():

516 for pd_name, pd in request.form.items():

496 if pd_name.startswith('category_'):

517 if pd_name.startswith('category_'):

497 category = pd_name[9:]

518 category = pd_name[9:]

@@ -514,14 +535,34 @@ def preferences():

514 safesearch = pd

535 safesearch = pd

515 elif pd_name.startswith('engine_'):

536 elif pd_name.startswith('engine_'):

516 if pd_name.find('__') > -1:

537 if pd_name.find('__') > -1:

538+ # TODO fix underscore vs space

539+ engine_name, category = [x.replace('_', ' ') for x in

517− engine_name, category = pd_name.replace('engine_', '', 1).split('__', 1)

540+ pd_name.replace('engine_', '', 1).split('__', 1)]

518 if engine_name in engines and category in engines[engine_name].categories:

541 if engine_name in engines and category in engines[engine_name].categories:

519 blocked_engines.append((engine_name, category))

542 blocked_engines.append((engine_name, category))

520 elif pd_name == 'theme':

543 elif pd_name == 'theme':

521 theme = pd if pd in themes else default_theme

544 theme = pd if pd in themes else default_theme

545+ elif pd_name.startswith('plugin_'):

546+ plugin_id = pd_name.replace('plugin_', '', 1)

547+ if not any(plugin.id == plugin_id for plugin in plugins):

548+ continue

549+ post_disabled_plugins.append(plugin_id)

522 else:

550 else:

523 resp.set_cookie(pd_name, pd, max_age=cookie_max_age)

551 resp.set_cookie(pd_name, pd, max_age=cookie_max_age)

524

552

553+ disabled_plugins = []

554+ allowed_plugins = []

555+ for plugin in plugins:

556+ if plugin.default_on:

557+ if plugin.id in post_disabled_plugins:

558+ disabled_plugins.append(plugin.id)

559+ elif plugin.id not in post_disabled_plugins:

560+ allowed_plugins.append(plugin.id)

561+

562+ resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age)

563+

564+ resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age)

565+

525 resp.set_cookie(

566 resp.set_cookie(

526 'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),

567 'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),

527 max_age=cookie_max_age

568 max_age=cookie_max_age

@@ -571,6 +612,8 @@ def preferences():

571 autocomplete_backends=autocomplete_backends,

612 autocomplete_backends=autocomplete_backends,

572 shortcuts={y: x for x, y in engine_shortcuts.items()},

613 shortcuts={y: x for x, y in engine_shortcuts.items()},

573 themes=themes,

614 themes=themes,

615+ plugins=plugins,

616+ allowed_plugins=[plugin.id for plugin in request.user_plugins],

574 theme=get_current_theme_name())

617 theme=get_current_theme_name())

575

618

576

619