on master help sign in
new preferences handling

Preferences class was introduced in order to handle user preferences. Right now it parses cookies and the form in preferences. Also it can retrieve settings based on the name of the setting. ATTENTION Please note that engine preferences are handled differently from now on. So it introduces incompatible changes. Every user who has saved preferences should reset and save his/her settings again. This change was needed, because everytime a default disabled engine was added saved user preferences would broke. Now engine setting tracking is fixed.

commitfe691a09888f6e6c6cc647de06f4eca0feb3451f authorNoemi Vanyi <sitbackandwait@gmail.com> dateFri 8 Apr 2016, 16:38 (11 years ago) parent9331fc2
6 files changed+316−168
searx/preferences.py+269 searx/search.py+5−10 searx/settings_robot.yml+1−1 searx/utils.py−23 searx/webapp.py+36−133 tests/unit/test_webapp.py+5−1
searx/preferences.pynew file+269
@@ -0,0 +1,269 @@
1+from searx import settings, autocomplete
2+from searx.languages import language_codes as languages
3+
4+
5+COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years
6+LANGUAGE_CODES = [l[0] for l in languages]
7+LANGUAGE_CODES.append('all')
8+DISABLED = 0
9+ENABLED = 1
10+
11+
12+class MissingArgumentException(Exception):
13+ pass
14+
15+
16+class ValidationException(Exception):
17+ pass
18+
19+
20+class Setting(object):
21+ """Base class of user settings"""
22+
23+ def __init__(self, default_value, **kwargs):
24+ super(Setting, self).__init__()
25+ self.value = default_value
26+ for key, value in kwargs.iteritems():
27+ setattr(self, key, value)
28+
29+ self._post_init()
30+
31+ def _post_init(self):
32+ pass
33+
34+ def parse(self, data):
35+ self.value = data
36+
37+ def get_value(self):
38+ return self.value
39+
40+ def save(self, name, resp):
41+ resp.set_cookie(name, bytes(self.value), max_age=COOKIE_MAX_AGE)
42+
43+
44+class StringSetting(Setting):
45+ """Setting of plain string values"""
46+ pass
47+
48+
49+class EnumStringSetting(Setting):
50+ """Setting of a value which can only come from the given choices"""
51+
52+ def _post_init(self):
53+ if not hasattr(self, 'choices'):
54+ raise MissingArgumentException('Missing argument: choices')
55+
56+ if self.value != '' and self.value not in self.choices:
57+ raise ValidationException('Invalid default value: {0}'.format(self.value))
58+
59+ def parse(self, data):
60+ if data not in self.choices and data != self.value:
61+ raise ValidationException('Invalid choice: {0}'.format(data))
62+ self.value = data
63+
64+
65+class MultipleChoiceSetting(EnumStringSetting):
66+ """Setting of values which can only come from the given choices"""
67+
68+ def _post_init(self):
69+ if not hasattr(self, 'choices'):
70+ raise MissingArgumentException('Missing argument: choices')
71+ for item in self.value:
72+ if item not in self.choices:
73+ raise ValidationException('Invalid default value: {0}'.format(self.value))
74+
75+ def parse(self, data):
76+ if data == '':
77+ self.value = []
78+ return
79+
80+ elements = data.split(',')
81+ for item in elements:
82+ if item not in self.choices:
83+ raise ValidationException('Invalid choice: {0}'.format(item))
84+ self.value = elements
85+
86+ def parse_form(self, data):
87+ self.value = []
88+ for choice in data:
89+ if choice in self.choices and choice not in self.value:
90+ self.value.append(choice)
91+
92+ def save(self, name, resp):
93+ resp.set_cookie(name, ','.join(self.value), max_age=COOKIE_MAX_AGE)
94+
95+
96+class MapSetting(Setting):
97+ """Setting of a value that has to be translated in order to be storable"""
98+
99+ def _post_init(self):
100+ if not hasattr(self, 'map'):
101+ raise MissingArgumentException('missing argument: map')
102+ if self.value not in self.map.values():
103+ raise ValidationException('Invalid default value')
104+
105+ def parse(self, data):
106+ if data not in self.map:
107+ raise ValidationException('Invalid choice: {0}'.format(data))
108+ self.value = self.map[data]
109+ self.key = data
110+
111+ def save(self, name, resp):
112+ resp.set_cookie(name, bytes(self.key), max_age=COOKIE_MAX_AGE)
113+
114+
115+class SwitchableSetting(Setting):
116+ """ Base class for settings that can be turned on && off"""
117+
118+ def _post_init(self):
119+ self.disabled = set()
120+ self.enabled = set()
121+ if not hasattr(self, 'choices'):
122+ raise MissingArgumentException('missing argument: choices')
123+
124+ def transform_form_items(self, items):
125+ return items
126+
127+ def transform_values(self, values):
128+ return values
129+
130+ def parse_cookie(self, data):
131+ if data[DISABLED] != '':
132+ self.disabled = set(data[DISABLED].split(','))
133+ if data[ENABLED] != '':
134+ self.enabled = set(data[ENABLED].split(','))
135+
136+ def parse_form(self, items):
137+ items = self.transform_form_items(items)
138+
139+ self.disabled = set()
140+ self.enabled = set()
141+ for choice in self.choices:
142+ if choice['default_on']:
143+ if choice['id'] in items:
144+ self.disabled.add(choice['id'])
145+ else:
146+ if choice['id'] not in items:
147+ self.enabled.add(choice['id'])
148+
149+ def save(self, resp):
150+ resp.set_cookie('disabled_{0}'.format(self.value), ','.join(self.disabled), max_age=COOKIE_MAX_AGE)
151+ resp.set_cookie('enabled_{0}'.format(self.value), ','.join(self.enabled), max_age=COOKIE_MAX_AGE)
152+
153+ def get_disabled(self):
154+ disabled = self.disabled
155+ for choice in self.choices:
156+ if not choice['default_on'] and choice['id'] not in self.enabled:
157+ disabled.add(choice['id'])
158+ return self.transform_values(disabled)
159+
160+ def get_enabled(self):
161+ enabled = self.enabled
162+ for choice in self.choices:
163+ if choice['default_on'] and choice['id'] not in self.disabled:
164+ enabled.add(choice['id'])
165+ return self.transform_values(enabled)
166+
167+
168+class EnginesSetting(SwitchableSetting):
169+ def _post_init(self):
170+ super(EnginesSetting, self)._post_init()
171+ transformed_choices = []
172+ for engine_name, engine in self.choices.iteritems():
173+ for category in engine.categories:
174+ transformed_choice = dict()
175+ transformed_choice['default_on'] = not engine.disabled
176+ transformed_choice['id'] = '{}__{}'.format(engine_name, category)
177+ transformed_choices.append(transformed_choice)
178+ self.choices = transformed_choices
179+
180+ def transform_form_items(self, items):
181+ return [item[len('engine_'):].replace('_', ' ').replace(' ', '__') for item in items]
182+
183+ def transform_values(self, values):
184+ if len(values) == 1 and values[0] == '':
185+ return list()
186+ transformed_values = []
187+ for value in values:
188+ engine, category = value.split('__')
189+ transformed_values.append((engine, category))
190+ return transformed_values
191+
192+
193+class PluginsSetting(SwitchableSetting):
194+ def _post_init(self):
195+ super(PluginsSetting, self)._post_init()
196+ transformed_choices = []
197+ for plugin in self.choices:
198+ transformed_choice = dict()
199+ transformed_choice['default_on'] = plugin.default_on
200+ transformed_choice['id'] = plugin.id
201+ transformed_choices.append(transformed_choice)
202+ self.choices = transformed_choices
203+
204+ def transform_form_items(self, items):
205+ return [item[len('plugin_'):] for item in items]
206+
207+
208+class Preferences(object):
209+ """Stores, validates and saves preferences to cookies"""
210+
211+ def __init__(self, themes, categories, engines, plugins):
212+ super(Preferences, self).__init__()
213+
214+ self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),
215+ 'language': EnumStringSetting('all', choices=LANGUAGE_CODES),
216+ 'locale': EnumStringSetting(settings['ui']['default_locale'],
217+ choices=settings['locales'].keys()),
218+ 'autocomplete': EnumStringSetting(settings['search']['autocomplete'],
219+ choices=autocomplete.backends.keys()),
220+ 'image_proxy': MapSetting(settings['server']['image_proxy'],
221+ map={'': settings['server']['image_proxy'],
222+ '0': False,
223+ '1': True}),
224+ 'method': EnumStringSetting('POST', choices=('GET', 'POST')),
225+ 'safesearch': MapSetting(settings['search']['safe_search'], map={'0': 0,
226+ '1': 1,
227+ '2': 2}),
228+ 'theme': EnumStringSetting(settings['ui']['default_theme'], choices=themes)}
229+
230+ self.engines = EnginesSetting('engines', choices=engines)
231+ self.plugins = PluginsSetting('plugins', choices=plugins)
232+
233+ def parse_cookies(self, input_data):
234+ for user_setting_name, user_setting in input_data.iteritems():
235+ if user_setting_name in self.key_value_settings:
236+ self.key_value_settings[user_setting_name].parse(user_setting)
237+ elif user_setting_name == 'disabled_engines':
238+ self.engines.parse_cookie([input_data['disabled_engines'], input_data['enabled_engines']])
239+ elif user_setting_name == 'disabled_plugins':
240+ self.plugins.parse_cookie([input_data['disabled_plugins'], input_data['enabled_plugins']])
241+
242+ def parse_form(self, input_data):
243+ disabled_engines = []
244+ enabled_categories = []
245+ disabled_plugins = []
246+ for user_setting_name, user_setting in input_data.iteritems():
247+ if user_setting_name in self.key_value_settings:
248+ self.key_value_settings[user_setting_name].parse(user_setting)
249+ elif user_setting_name.startswith('engine_'):
250+ disabled_engines.append(user_setting_name)
251+ elif user_setting_name.startswith('category_'):
252+ enabled_categories.append(user_setting_name[len('category_'):])
253+ elif user_setting_name.startswith('plugin_'):
254+ disabled_plugins.append(user_setting_name)
255+ self.key_value_settings['categories'].parse_form(enabled_categories)
256+ self.engines.parse_form(disabled_engines)
257+ self.plugins.parse_form(disabled_plugins)
258+
259+ # cannot be used in case of engines or plugins
260+ def get_value(self, user_setting_name):
261+ if user_setting_name in self.key_value_settings:
262+ return self.key_value_settings[user_setting_name].get_value()
263+
264+ def save(self, resp):
265+ for user_setting_name, user_setting in self.key_value_settings.iteritems():
266+ user_setting.save(user_setting_name, resp)
267+ self.engines.save(resp)
268+ self.plugins.save(resp)
269+ return resp
searx/search.py+5−10
@@ -23,7 +23,7 @@ from searx.engines import (
2323 categories, engines
2424)
2525from searx.languages import language_codes
26−from searx.utils import gen_useragent, get_blocked_engines
26+from searx.utils import gen_useragent
2727from searx.query import Query
2828from searx.results import ResultContainer
2929from searx import logger
@@ -140,15 +140,13 @@ class Search(object):
140140 self.lang = 'all'
141141
142142 # set blocked engines
143− self.blocked_engines = get_blocked_engines(engines, request.cookies)
143+ self.blocked_engines = request.preferences.engines.get_disabled()
144144
145145 self.result_container = ResultContainer()
146146 self.request_data = {}
147147
148148 # set specific language if set
149− if request.cookies.get('language')\
150− and request.cookies['language'] in (x[0] for x in language_codes):
151− self.lang = request.cookies['language']
149+ self.lang = request.preferences.get_value('language')
152150
153151 # set request method
154152 if request.method == 'POST':
@@ -294,11 +292,8 @@ class Search(object):
294292 else:
295293 request_params['language'] = self.lang
296294
297− try:
298− # 0 = None, 1 = Moderate, 2 = Strict
299− request_params['safesearch'] = int(request.cookies.get('safesearch'))
300− except Exception:
301− request_params['safesearch'] = settings['search']['safe_search']
295+ # 0 = None, 1 = Moderate, 2 = Strict
296+ request_params['safesearch'] = request.preferences.get_value('safesearch')
302297
303298 # update request parameters dependent on
304299 # search-engine (contained in engines folder)
searx/settings_robot.yml+1−1
@@ -4,7 +4,7 @@ general:
44
55search:
66 safe_search : 0
7− autocomplete : 0
7+ autocomplete : ""
88
99server:
1010 port : 11111
searx/utils.py−23
@@ -230,26 +230,3 @@ def list_get(a_list, index, default=None):
230230 return a_list[index]
231231 else:
232232 return default
233−
234−
235−def get_blocked_engines(engines, cookies):
236− if 'blocked_engines' not in cookies:
237− return [(engine_name, category) for engine_name in engines
238− for category in engines[engine_name].categories if engines[engine_name].disabled]
239−
240− blocked_engine_strings = cookies.get('blocked_engines', '').split(',')
241− blocked_engines = []
242−
243− if not blocked_engine_strings:
244− return blocked_engines
245−
246− for engine_string in blocked_engine_strings:
247− if engine_string.find('__') > -1:
248− engine, category = engine_string.split('__', 1)
249− if engine in engines and category in engines[engine].categories:
250− blocked_engines.append((engine, category))
251− elif engine_string in engines:
252− for category in engines[engine_string].categories:
253− blocked_engines.append((engine_string, category))
254−
255− return blocked_engines
searx/webapp.py+36−133
@@ -56,7 +56,7 @@ from searx.engines import (
5656from searx.utils import (
5757 UnicodeWriter, highlight_content, html_to_text, get_themes,
5858 get_static_files, get_result_templates, gen_useragent, dict_subset,
59− prettify_url, get_blocked_engines
59+ prettify_url
6060)
6161from searx.version import VERSION_STRING
6262from searx.languages import language_codes
@@ -64,6 +64,7 @@ from searx.search import Search
6464from searx.query import Query
6565from searx.autocomplete import searx_bang, backends as autocomplete_backends
6666from searx.plugins import plugins
67+from searx.preferences import Preferences
6768
6869# check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.
6970# They are needed for SSL connection without trouble, see #298
@@ -109,8 +110,6 @@ for indice, theme in enumerate(themes):
109110 for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
110111 global_favicons[indice].extend(filenames)
111112
112−cookie_max_age = 60 * 60 * 24 * 365 * 5 # 5 years
113−
114113_category_names = (gettext('files'),
115114 gettext('general'),
116115 gettext('music'),
@@ -222,9 +221,7 @@ def get_current_theme_name(override=None):
222221
223222 if override and override in themes:
224223 return override
225− theme_name = request.args.get('theme',
226− request.cookies.get('theme',
227− default_theme))
224+ theme_name = request.args.get('theme', request.preferences.get_value('theme'))
228225 if theme_name not in themes:
229226 theme_name = default_theme
230227 return theme_name
@@ -262,12 +259,8 @@ def image_proxify(url):
262259
263260
264261def render(template_name, override_theme=None, **kwargs):
265− blocked_engines = get_blocked_engines(engines, request.cookies)
266−
267− autocomplete = request.cookies.get('autocomplete', settings['search']['autocomplete'])
268−
269− if autocomplete not in autocomplete_backends:
270− autocomplete = None
262+ blocked_engines = request.preferences.engines.get_disabled()
263+ autocomplete = request.preferences.get_value('autocomplete')
271264
272265 nonblocked_categories = set(category for engine_name in engines
273266 for category in engines[engine_name].categories
@@ -295,7 +288,7 @@ def render(template_name, override_theme=None, **kwargs):
295288 kwargs['selected_categories'].append(c)
296289
297290 if not kwargs['selected_categories']:
298− cookie_categories = request.cookies.get('categories', '').split(',')
291+ cookie_categories = request.preferences.get_value('categories')
299292 for ccateg in cookie_categories:
300293 if ccateg in categories:
301294 kwargs['selected_categories'].append(ccateg)
@@ -311,9 +304,9 @@ def render(template_name, override_theme=None, **kwargs):
311304
312305 kwargs['searx_version'] = VERSION_STRING
313306
314− kwargs['method'] = request.cookies.get('method', 'POST')
307+ kwargs['method'] = request.preferences.get_value('method')
315308
316− kwargs['safesearch'] = request.cookies.get('safesearch', str(settings['search']['safe_search']))
309+ kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
317310
318311 # override url_for function in templates
319312 kwargs['url_for'] = url_for_theme
@@ -347,14 +340,18 @@ def render(template_name, override_theme=None, **kwargs):
347340@app.before_request
348341def pre_request():
349342 # merge GET, POST vars
343+ preferences = Preferences(themes, categories.keys(), engines, plugins)
344+ preferences.parse_cookies(request.cookies)
345+ request.preferences = preferences
346+
350347 request.form = dict(request.form.items())
351348 for k, v in request.args.items():
352349 if k not in request.form:
353350 request.form[k] = v
354351
355352 request.user_plugins = []
356− allowed_plugins = request.cookies.get('allowed_plugins', '').split(',')
357− disabled_plugins = request.cookies.get('disabled_plugins', '').split(',')
353+ allowed_plugins = preferences.plugins.get_enabled()
354+ disabled_plugins = preferences.plugins.get_disabled()
358355 for plugin in plugins:
359356 if ((plugin.default_on and plugin.id not in disabled_plugins)
360357 or plugin.id in allowed_plugins):
@@ -486,7 +483,7 @@ def autocompleter():
486483 request_data = request.args
487484
488485 # set blocked engines
489− blocked_engines = get_blocked_engines(engines, request.cookies)
486+ blocked_engines = request.preferences.engines.get_disabled()
490487
491488 # parse query
492489 query = Query(request_data.get('q', '').encode('utf-8'), blocked_engines)
@@ -496,8 +493,8 @@ def autocompleter():
496493 if not query.getSearchQuery():
497494 return '', 400
498495
499− # get autocompleter
500− completer = autocomplete_backends.get(request.cookies.get('autocomplete', settings['search']['autocomplete']))
496+ # run autocompleter
497+ completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
501498
502499 # parse searx specific autocompleter results like !bang
503500 raw_results = searx_bang(query)
@@ -532,117 +529,23 @@ def autocompleter():
532529
533530@app.route('/preferences', methods=['GET', 'POST'])
534531def preferences():
535− """Render preferences page.
536−
537− Settings that are going to be saved as cookies."""
538− lang = None
539− image_proxy = request.cookies.get('image_proxy', settings['server'].get('image_proxy'))
540−
541− if request.cookies.get('language')\
542− and request.cookies['language'] in (x[0] for x in language_codes):
543− lang = request.cookies['language']
544−
545− blocked_engines = []
546−
547− resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
548−
549− if request.method == 'GET':
550− blocked_engines = get_blocked_engines(engines, request.cookies)
551− else: # on save
552− selected_categories = []
553− post_disabled_plugins = []
554− locale = None
555− autocomplete = ''
556− method = 'POST'
557− safesearch = settings['search']['safe_search']
558− for pd_name, pd in request.form.items():
559− if pd_name.startswith('category_'):
560− category = pd_name[9:]
561− if category not in categories:
562− continue
563− selected_categories.append(category)
564− elif pd_name == 'locale' and pd in settings['locales']:
565− locale = pd
566− elif pd_name == 'image_proxy':
567− image_proxy = pd
568− elif pd_name == 'autocomplete':
569− autocomplete = pd
570− elif pd_name == 'language' and (pd == 'all' or
571− pd in (x[0] for
572− x in language_codes)):
573− lang = pd
574− elif pd_name == 'method':
575− method = pd
576− elif pd_name == 'safesearch':
577− safesearch = pd
578− elif pd_name.startswith('engine_'):
579− if pd_name.find('__') > -1:
580− # TODO fix underscore vs space
581− engine_name, category = [x.replace('_', ' ') for x in
582− pd_name.replace('engine_', '', 1).split('__', 1)]
583− if engine_name in engines and category in engines[engine_name].categories:
584− blocked_engines.append((engine_name, category))
585− elif pd_name == 'theme':
586− theme = pd if pd in themes else default_theme
587− elif pd_name.startswith('plugin_'):
588− plugin_id = pd_name.replace('plugin_', '', 1)
589− if not any(plugin.id == plugin_id for plugin in plugins):
590− continue
591− post_disabled_plugins.append(plugin_id)
592− else:
593− resp.set_cookie(pd_name, pd, max_age=cookie_max_age)
594−
595− disabled_plugins = []
596− allowed_plugins = []
597− for plugin in plugins:
598− if plugin.default_on:
599− if plugin.id in post_disabled_plugins:
600− disabled_plugins.append(plugin.id)
601− elif plugin.id not in post_disabled_plugins:
602− allowed_plugins.append(plugin.id)
603−
604− resp.set_cookie('disabled_plugins', ','.join(disabled_plugins), max_age=cookie_max_age)
532+ """Render preferences page && save user preferences"""
605533
606− resp.set_cookie('allowed_plugins', ','.join(allowed_plugins), max_age=cookie_max_age)
607−
608− resp.set_cookie(
609− 'blocked_engines', ','.join('__'.join(e) for e in blocked_engines),
610− max_age=cookie_max_age
611− )
612−
613− if locale:
614− resp.set_cookie(
615− 'locale', locale,
616− max_age=cookie_max_age
617− )
618−
619− if lang:
620− resp.set_cookie(
621− 'language', lang,
622− max_age=cookie_max_age
623− )
624−
625− if selected_categories:
626− # cookie max age: 4 weeks
627− resp.set_cookie(
628− 'categories', ','.join(selected_categories),
629− max_age=cookie_max_age
630− )
631−
632− resp.set_cookie(
633− 'autocomplete', autocomplete,
634− max_age=cookie_max_age
635− )
636−
637− resp.set_cookie('method', method, max_age=cookie_max_age)
638−
639− resp.set_cookie('safesearch', str(safesearch), max_age=cookie_max_age)
640−
641− resp.set_cookie('image_proxy', image_proxy, max_age=cookie_max_age)
642−
643− resp.set_cookie('theme', theme, max_age=cookie_max_age)
644−
645− return resp
534+ # save preferences
535+ if request.method == 'POST':
536+ resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
537+ try:
538+ request.preferences.parse_form(request.form)
539+ except ValidationException:
540+ # TODO use flash feature of flask
541+ return resp
542+ return request.preferences.save(resp)
543+
544+ # render preferences
545+ image_proxy = request.preferences.get_value('image_proxy')
546+ lang = request.preferences.get_value('language')
547+ blocked_engines = request.preferences.engines.get_disabled()
548+ allowed_plugins = request.preferences.plugins.get_enabled()
646549
647550 # stats for preferences page
648551 stats = {}
@@ -664,7 +567,7 @@ def preferences():
664567 return render('preferences.html',
665568 locales=settings['locales'],
666569 current_locale=get_locale(),
667− current_language=lang or 'all',
570+ current_language=lang,
668571 image_proxy=image_proxy,
669572 language_codes=language_codes,
670573 engines_by_category=categories,
@@ -674,7 +577,7 @@ def preferences():
674577 shortcuts={y: x for x, y in engine_shortcuts.items()},
675578 themes=themes,
676579 plugins=plugins,
677− allowed_plugins=[plugin.id for plugin in request.user_plugins],
580+ allowed_plugins=allowed_plugins,
678581 theme=get_current_theme_name())
679582
680583
@@ -750,7 +653,7 @@ Disallow: /preferences
750653def opensearch():
751654 method = 'post'
752655
753− if request.cookies.get('method', 'POST') == 'GET':
656+ if request.preferences.get_value('method') == 'GET':
754657 method = 'get'
755658
756659 # chrome/chromium only supports HTTP GET....
tests/unit/test_webapp.py+5−1
@@ -12,7 +12,6 @@ class ViewsTestCase(SearxTestCase):
1212 def setUp(self):
1313 webapp.app.config['TESTING'] = True # to get better error messages
1414 self.app = webapp.app.test_client()
15− webapp.default_theme = 'default'
1615
1716 # set some defaults
1817 self.test_results = [
@@ -43,6 +42,11 @@ class ViewsTestCase(SearxTestCase):
4342
4443 webapp.Search.search = search_mock
4544
45+ def get_current_theme_name_mock(override=None):
46+ return 'default'
47+
48+ webapp.get_current_theme_name = get_current_theme_name_mock
49+
4650 self.maxDiff = None # to see full diffs
4751
4852 def test_index_empty(self):