on master help sign in
Merge pull request #2291 from dalf/settings2

[enh] user settings can relied on the default settings

commita1e6bc4cee2690eb7f2103901ada9a50846a7419 authorAlexandre Flament <alex@al-f.net> dateTue 1 Dec 2020, 14:57 (6 years ago) parent0e02feeb4b81a5
13 files changed+499−28
docs/admin/settings.rst+77 docs/blog/private-engines.rst+2 searx/__init__.py+3−28 searx/exceptions.py+8 searx/settings_loader.py+129 tests/unit/settings/empty_settings.yml tests/unit/settings/syntaxerror_settings.yml+2 tests/unit/settings/user_settings.yml+111 tests/unit/settings/user_settings_keep_only.yml+14 tests/unit/settings/user_settings_remove.yml+10 tests/unit/settings/user_settings_remove2.yml+15 tests/unit/settings/user_settings_simple.yml+6 tests/unit/test_settings_loader.py+122
docs/admin/settings.rst+77
@@ -206,3 +206,80 @@ Engine settings
206206
207207 A few more options are possible, but they are pretty specific to some
208208 engines, and so won't be described here.
209+
210+
211+.. _settings location:
212+
213+settings.yml location
214+=====================
215+
216+First, searx will try to load settings.yml from these locations:
217+
218+1. the full path specified in the ``SEARX_SETTINGS_PATH`` environment variable.
219+2. ``/etc/searx/settings.yml``
220+
221+If these files don't exist (or are empty or can't be read), searx uses the :origin:`searx/settings.yml` file.
222+
223+.. _ settings use_default_settings:
224+
225+use_default_settings
226+====================
227+
228+.. note::
229+
230+ If searx is cloned from a git repository, most probably there is no need to have an user settings.
231+
232+The user defined settings.yml can relied on the default configuration :origin:`searx/settings.yml` using ``use_default_settings: True``.
233+
234+In the following example, the actual settings are the default settings defined in :origin:`searx/settings.yml` with the exception of the ``secret_key`` and the ``bind_address``:
235+
236+.. code-block:: yaml
237+
238+ use_default_settings: True
239+ server:
240+ secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
241+ bind_address: "0.0.0.0"
242+
243+With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.
244+
245+In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:
246+
247+.. code-block:: yaml
248+
249+ use_default_settings: True
250+ server:
251+ secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
252+ engines:
253+ - name: arch linux wiki
254+ tokens: ['$ecretValue']
255+
256+It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:
257+
258+.. code-block:: yaml
259+
260+ use_default_settings:
261+ engines:
262+ remove:
263+ - google
264+ server:
265+ secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
266+ engines:
267+ - name: arch linux wiki
268+ tokens: ['$ecretValue']
269+
270+As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
271+
272+.. code-block:: yaml
273+
274+ use_default_settings:
275+ engines:
276+ keep_only:
277+ - google
278+ - duckduckgo
279+ server:
280+ secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
281+ engines:
282+ - name: google
283+ tokens: ['$ecretValue']
284+ - name: duckduckgo
285+ tokens: ['$ecretValue']
docs/blog/private-engines.rst+2
@@ -7,6 +7,8 @@ enabled engines on their instances. It might be because they do not want to
77expose some private information through an offline engine. Or they
88would rather share engines only with their trusted friends or colleagues.
99
10+.. _private engines:
11+
1012Private engines
1113===============
1214
searx/__init__.py+3−28
@@ -16,39 +16,15 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
1616'''
1717
1818import logging
19+import searx.settings_loader
1920from os import environ
2021from os.path import realpath, dirname, join, abspath, isfile
21−from io import open
22−from yaml import safe_load
2322
2423
2524searx_dir = abspath(dirname(__file__))
2625engine_dir = dirname(realpath(__file__))
2726static_path = abspath(join(dirname(__file__), 'static'))
28−
29−
30−def check_settings_yml(file_name):
31− if isfile(file_name):
32− return file_name
33− else:
34− return None
35−
36−
37−# find location of settings.yml
38−if 'SEARX_SETTINGS_PATH' in environ:
39− # if possible set path to settings using the
40− # enviroment variable SEARX_SETTINGS_PATH
41− settings_path = check_settings_yml(environ['SEARX_SETTINGS_PATH'])
42−else:
43− # if not, get it from searx code base or last solution from /etc/searx
44− settings_path = check_settings_yml(join(searx_dir, 'settings.yml')) or check_settings_yml('/etc/searx/settings.yml')
45−
46−if not settings_path:
47− raise Exception('settings.yml not found')
48−
49−# load settings
50−with open(settings_path, 'r', encoding='utf-8') as settings_yaml:
51− settings = safe_load(settings_yaml)
27+settings, settings_load_message = searx.settings_loader.load_settings()
5228
5329if settings['ui']['static_path']:
5430 static_path = settings['ui']['static_path']
@@ -58,7 +34,6 @@ enable debug if
5834the environnement variable SEARX_DEBUG is 1 or true
5935(whatever the value in settings.yml)
6036or general.debug=True in settings.yml
61−
6237disable debug if
6338the environnement variable SEARX_DEBUG is 0 or false
6439(whatever the value in settings.yml)
@@ -78,7 +53,7 @@ else:
7853 logging.basicConfig(level=logging.WARNING)
7954
8055logger = logging.getLogger('searx')
81−logger.debug('read configuration from %s', settings_path)
56+logger.info(settings_load_message)
8257logger.info('Initialisation done')
8358
8459if 'SEARX_SECRET' in environ:
searx/exceptions.py+8
@@ -31,3 +31,11 @@ class SearxParameterException(SearxException):
3131 self.message = message
3232 self.parameter_name = name
3333 self.parameter_value = value
34+
35+
36+class SearxSettingsException(SearxException):
37+
38+ def __init__(self, message, filename):
39+ super().__init__(message)
40+ self.message = message
41+ self.filename = filename
searx/settings_loader.pynew file+129
@@ -0,0 +1,129 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+from os import environ
4+from os.path import dirname, join, abspath, isfile
5+from collections.abc import Mapping
6+from itertools import filterfalse
7+
8+import yaml
9+
10+from searx.exceptions import SearxSettingsException
11+
12+
13+searx_dir = abspath(dirname(__file__))
14+
15+
16+def check_settings_yml(file_name):
17+ if isfile(file_name):
18+ return file_name
19+ return None
20+
21+
22+def load_yaml(file_name):
23+ try:
24+ with open(file_name, 'r', encoding='utf-8') as settings_yaml:
25+ return yaml.safe_load(settings_yaml)
26+ except IOError as e:
27+ raise SearxSettingsException(e, file_name)
28+ except yaml.YAMLError as e:
29+ raise SearxSettingsException(e, file_name)
30+
31+
32+def get_default_settings_path():
33+ return check_settings_yml(join(searx_dir, 'settings.yml'))
34+
35+
36+def get_user_settings_path():
37+ # find location of settings.yml
38+ if 'SEARX_SETTINGS_PATH' in environ:
39+ # if possible set path to settings using the
40+ # enviroment variable SEARX_SETTINGS_PATH
41+ return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
42+
43+ # if not, get it from searx code base or last solution from /etc/searx
44+ return check_settings_yml('/etc/searx/settings.yml')
45+
46+
47+def update_dict(default_dict, user_dict):
48+ for k, v in user_dict.items():
49+ if isinstance(v, Mapping):
50+ default_dict[k] = update_dict(default_dict.get(k, {}), v)
51+ else:
52+ default_dict[k] = v
53+ return default_dict
54+
55+
56+def update_settings(default_settings, user_settings):
57+ # merge everything except the engines
58+ for k, v in user_settings.items():
59+ if k not in ('use_default_settings', 'engines'):
60+ update_dict(default_settings[k], v)
61+
62+ # parse the engines
63+ remove_engines = None
64+ keep_only_engines = None
65+ use_default_settings = user_settings.get('use_default_settings')
66+ if isinstance(use_default_settings, dict):
67+ remove_engines = use_default_settings.get('engines', {}).get('remove')
68+ keep_only_engines = use_default_settings.get('engines', {}).get('keep_only')
69+
70+ if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None:
71+ engines = default_settings['engines']
72+
73+ # parse "use_default_settings.engines.remove"
74+ if remove_engines is not None:
75+ engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines))
76+
77+ # parse "use_default_settings.engines.keep_only"
78+ if keep_only_engines is not None:
79+ engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines))
80+
81+ # parse "engines"
82+ user_engines = user_settings.get('engines')
83+ if user_engines:
84+ engines_dict = dict((definition['name'], definition) for definition in engines)
85+ for user_engine in user_engines:
86+ default_engine = engines_dict.get(user_engine['name'])
87+ if default_engine:
88+ update_dict(default_engine, user_engine)
89+ else:
90+ engines.append(user_engine)
91+
92+ # store the result
93+ default_settings['engines'] = engines
94+
95+ return default_settings
96+
97+
98+def is_use_default_settings(user_settings):
99+ use_default_settings = user_settings.get('use_default_settings')
100+ if use_default_settings is True:
101+ return True
102+ if isinstance(use_default_settings, dict):
103+ return True
104+ if use_default_settings is False or use_default_settings is None:
105+ return False
106+ raise ValueError('Invalid value for use_default_settings')
107+
108+
109+def load_settings(load_user_setttings=True):
110+ default_settings_path = get_default_settings_path()
111+ user_settings_path = get_user_settings_path()
112+ if user_settings_path is None or not load_user_setttings:
113+ # no user settings
114+ return (load_yaml(default_settings_path),
115+ 'load the default settings from {}'.format(default_settings_path))
116+
117+ # user settings
118+ user_settings = load_yaml(user_settings_path)
119+ if is_use_default_settings(user_settings):
120+ # the user settings are merged with the default configuration
121+ default_settings = load_yaml(default_settings_path)
122+ update_settings(default_settings, user_settings)
123+ return (default_settings,
124+ 'merge the default settings ( {} ) and the user setttings ( {} )'
125+ .format(default_settings_path, user_settings_path))
126+
127+ # the user settings, fully replace the default configuration
128+ return (user_settings,
129+ 'load the user settings from {}'.format(user_settings_path))
tests/unit/settings/empty_settings.ymlnew file

No changes to the content.

tests/unit/settings/syntaxerror_settings.ymlnew file+2
@@ -0,0 +1,2 @@
1+Test:
2+ **********
tests/unit/settings/user_settings.ymlnew file+111
@@ -0,0 +1,111 @@
1+general:
2+ debug : False
3+ instance_name : "searx"
4+
5+search:
6+ safe_search : 0
7+ autocomplete : ""
8+ default_lang : ""
9+ ban_time_on_fail : 5
10+ max_ban_time_on_fail : 120
11+
12+server:
13+ port : 9000
14+ bind_address : "0.0.0.0"
15+ secret_key : "user_settings_secret"
16+ base_url : False
17+ image_proxy : False
18+ http_protocol_version : "1.0"
19+ method: "POST"
20+ default_http_headers:
21+ X-Content-Type-Options : nosniff
22+ X-XSS-Protection : 1; mode=block
23+ X-Download-Options : noopen
24+ X-Robots-Tag : noindex, nofollow
25+ Referrer-Policy : no-referrer
26+
27+ui:
28+ static_path : ""
29+ templates_path : ""
30+ default_theme : oscar
31+ default_locale : ""
32+ theme_args :
33+ oscar_style : logicodev
34+
35+engines:
36+ - name : wikidata
37+ engine : wikidata
38+ shortcut : wd
39+ timeout : 3.0
40+ weight : 2
41+
42+ - name : wikibooks
43+ engine : mediawiki
44+ shortcut : wb
45+ categories : general
46+ base_url : "https://{language}.wikibooks.org/"
47+ number_of_results : 5
48+ search_type : text
49+
50+ - name : wikinews
51+ engine : mediawiki
52+ shortcut : wn
53+ categories : news
54+ base_url : "https://{language}.wikinews.org/"
55+ number_of_results : 5
56+ search_type : text
57+
58+ - name : wikiquote
59+ engine : mediawiki
60+ shortcut : wq
61+ categories : general
62+ base_url : "https://{language}.wikiquote.org/"
63+ number_of_results : 5
64+ search_type : text
65+
66+locales:
67+ en : English
68+ ar : العَرَبِيَّة (Arabic)
69+ bg : Български (Bulgarian)
70+ bo : བོད་སྐད་ (Tibetian)
71+ ca : Català (Catalan)
72+ cs : Čeština (Czech)
73+ cy : Cymraeg (Welsh)
74+ da : Dansk (Danish)
75+ de : Deutsch (German)
76+ el_GR : Ελληνικά (Greek_Greece)
77+ eo : Esperanto (Esperanto)
78+ es : Español (Spanish)
79+ et : Eesti (Estonian)
80+ eu : Euskara (Basque)
81+ fa_IR : (fārsī) فارسى (Persian)
82+ fi : Suomi (Finnish)
83+ fil : Wikang Filipino (Filipino)
84+ fr : Français (French)
85+ gl : Galego (Galician)
86+ he : עברית (Hebrew)
87+ hr : Hrvatski (Croatian)
88+ hu : Magyar (Hungarian)
89+ ia : Interlingua (Interlingua)
90+ it : Italiano (Italian)
91+ ja : 日本語 (Japanese)
92+ lt : Lietuvių (Lithuanian)
93+ nl : Nederlands (Dutch)
94+ nl_BE : Vlaams (Dutch_Belgium)
95+ oc : Lenga D'òc (Occitan)
96+ pl : Polski (Polish)
97+ pt : Português (Portuguese)
98+ pt_BR : Português (Portuguese_Brazil)
99+ ro : Română (Romanian)
100+ ru : Русский (Russian)
101+ sk : Slovenčina (Slovak)
102+ sl : Slovenski (Slovene)
103+ sr : српски (Serbian)
104+ sv : Svenska (Swedish)
105+ te : తెలుగు (telugu)
106+ ta : தமிழ் (Tamil)
107+ tr : Türkçe (Turkish)
108+ uk : українська мова (Ukrainian)
109+ vi : tiếng việt (Vietnamese)
110+ zh : 中文 (Chinese)
111+ zh_TW : 國語 (Taiwanese Mandarin)
tests/unit/settings/user_settings_keep_only.ymlnew file+14
@@ -0,0 +1,14 @@
1+use_default_settings:
2+ engines:
3+ keep_only:
4+ - wikibooks
5+ - wikinews
6+server:
7+ secret_key: "user_secret_key"
8+ bind_address: "0.0.0.0"
9+ default_http_headers:
10+ Custom-Header: Custom-Value
11+engines:
12+ - name: wikipedia
13+ - name: newengine
14+ engine: dummy
tests/unit/settings/user_settings_remove.ymlnew file+10
@@ -0,0 +1,10 @@
1+use_default_settings:
2+ engines:
3+ remove:
4+ - wikibooks
5+ - wikinews
6+server:
7+ secret_key: "user_secret_key"
8+ bind_address: "0.0.0.0"
9+ default_http_headers:
10+ Custom-Header: Custom-Value
tests/unit/settings/user_settings_remove2.ymlnew file+15
@@ -0,0 +1,15 @@
1+use_default_settings:
2+ engines:
3+ remove:
4+ - wikibooks
5+ - wikinews
6+server:
7+ secret_key: "user_secret_key"
8+ bind_address: "0.0.0.0"
9+ default_http_headers:
10+ Custom-Header: Custom-Value
11+engines:
12+ - name: wikipedia
13+ tokens: ['secret_token']
14+ - name: newengine
15+ engine: dummy
tests/unit/settings/user_settings_simple.ymlnew file+6
@@ -0,0 +1,6 @@
1+use_default_settings: True
2+server:
3+ secret_key: "user_secret_key"
4+ bind_address: "0.0.0.0"
5+ default_http_headers:
6+ Custom-Header: Custom-Value
tests/unit/test_settings_loader.pynew file+122
@@ -0,0 +1,122 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+from os.path import dirname, join, abspath
4+from unittest.mock import patch
5+
6+from searx.testing import SearxTestCase
7+from searx.exceptions import SearxSettingsException
8+from searx import settings_loader
9+
10+
11+test_dir = abspath(dirname(__file__))
12+
13+
14+class TestLoad(SearxTestCase):
15+
16+ def test_load_zero(self):
17+ with self.assertRaises(SearxSettingsException):
18+ settings_loader.load_yaml('/dev/zero')
19+
20+ with self.assertRaises(SearxSettingsException):
21+ settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml'))
22+
23+ with self.assertRaises(SearxSettingsException):
24+ settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml'))
25+
26+ def test_check_settings_yml(self):
27+ self.assertIsNone(settings_loader.check_settings_yml('/dev/zero'))
28+
29+ bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml')
30+ self.assertEqual(settings_loader.check_settings_yml(bad_settings_path), bad_settings_path)
31+
32+
33+class TestDefaultSettings(SearxTestCase):
34+
35+ def test_load(self):
36+ settings, msg = settings_loader.load_settings(load_user_setttings=False)
37+ self.assertTrue(msg.startswith('load the default settings from'))
38+ self.assertFalse(settings['general']['debug'])
39+ self.assertTrue(isinstance(settings['general']['instance_name'], str))
40+ self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
41+ self.assertTrue(isinstance(settings['server']['port'], int))
42+ self.assertTrue(isinstance(settings['server']['bind_address'], str))
43+ self.assertTrue(isinstance(settings['engines'], list))
44+ self.assertTrue(isinstance(settings['locales'], dict))
45+ self.assertTrue(isinstance(settings['doi_resolvers'], dict))
46+ self.assertTrue(isinstance(settings['default_doi_resolver'], str))
47+
48+
49+class TestUserSettings(SearxTestCase):
50+
51+ def test_is_use_default_settings(self):
52+ self.assertFalse(settings_loader.is_use_default_settings({}))
53+ self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': True}))
54+ self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': {}}))
55+ with self.assertRaises(ValueError):
56+ self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 1}))
57+ with self.assertRaises(ValueError):
58+ self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0}))
59+
60+ def test_user_settings_not_found(self):
61+ with patch.dict(settings_loader.environ,
62+ {'SEARX_SETTINGS_PATH': '/dev/null'}):
63+ settings, msg = settings_loader.load_settings()
64+ self.assertTrue(msg.startswith('load the default settings from'))
65+ self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
66+
67+ def test_user_settings(self):
68+ with patch.dict(settings_loader.environ,
69+ {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}):
70+ settings, msg = settings_loader.load_settings()
71+ self.assertTrue(msg.startswith('merge the default settings'))
72+ self.assertEqual(settings['server']['secret_key'], "user_secret_key")
73+ self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
74+
75+ def test_user_settings_remove(self):
76+ with patch.dict(settings_loader.environ,
77+ {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}):
78+ settings, msg = settings_loader.load_settings()
79+ self.assertTrue(msg.startswith('merge the default settings'))
80+ self.assertEqual(settings['server']['secret_key'], "user_secret_key")
81+ self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
82+ engine_names = [engine['name'] for engine in settings['engines']]
83+ self.assertNotIn('wikinews', engine_names)
84+ self.assertNotIn('wikibooks', engine_names)
85+ self.assertIn('wikipedia', engine_names)
86+
87+ def test_user_settings_remove2(self):
88+ with patch.dict(settings_loader.environ,
89+ {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}):
90+ settings, msg = settings_loader.load_settings()
91+ self.assertTrue(msg.startswith('merge the default settings'))
92+ self.assertEqual(settings['server']['secret_key'], "user_secret_key")
93+ self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
94+ engine_names = [engine['name'] for engine in settings['engines']]
95+ self.assertNotIn('wikinews', engine_names)
96+ self.assertNotIn('wikibooks', engine_names)
97+ self.assertIn('wikipedia', engine_names)
98+ wikipedia = list(filter(lambda engine: (engine.get('name')) == 'wikipedia', settings['engines']))
99+ self.assertEqual(wikipedia[0]['engine'], 'wikipedia')
100+ self.assertEqual(wikipedia[0]['tokens'], ['secret_token'])
101+ newengine = list(filter(lambda engine: (engine.get('name')) == 'newengine', settings['engines']))
102+ self.assertEqual(newengine[0]['engine'], 'dummy')
103+
104+ def test_user_settings_keep_only(self):
105+ with patch.dict(settings_loader.environ,
106+ {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}):
107+ settings, msg = settings_loader.load_settings()
108+ self.assertTrue(msg.startswith('merge the default settings'))
109+ engine_names = [engine['name'] for engine in settings['engines']]
110+ self.assertEqual(engine_names, ['wikibooks', 'wikinews', 'wikipedia', 'newengine'])
111+ # wikipedia has been removed, then added again with the "engine" section of user_settings_keep_only.yml
112+ self.assertEqual(len(settings['engines'][2]), 1)
113+
114+ def test_custom_settings(self):
115+ with patch.dict(settings_loader.environ,
116+ {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}):
117+ settings, msg = settings_loader.load_settings()
118+ self.assertTrue(msg.startswith('load the user settings from'))
119+ self.assertEqual(settings['server']['port'], 9000)
120+ self.assertEqual(settings['server']['secret_key'], "user_settings_secret")
121+ engine_names = [engine['name'] for engine in settings['engines']]
122+ self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote'])