git.iko.soy/searx
on master help sign in
Merge branch 'master' of https://github.com/asciimoo/searx into filtron

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

commit4d67164918db9bd9f2a5a58765e6082a633f3e8e authorMarkus Heiser <markus.heiser@darmarit.de> dateSun 23 Feb 2020, 13:50 (7 years ago) parent5fb6d4f8c26097
7 files changed+84−17
compare with master
unifiedsplit
docs/conf.py+1−1 docs/dev/reST.rst+1−1 docs/index.rst+4−2 searx/engines/etools.py+54 searx/preferences.py+7−2 searx/settings.yml+6 searx/webapp.py+11−11
docs/conf.py+1−1
@@ -12,7 +12,7 @@ DOCS_URL = os.environ.get("DOCS_URL", "https://asciimoo.github.io/searx/")
1212# Project --------------------------------------------------------------
1313
1414project = u'searx'
15−copyright = u'2015-2019, Adam Tauber, Noémi Ványi'
15+copyright = u'2015-2020, Adam Tauber, Noémi Ványi'
1616author = u'Adam Tauber'
1717release, version = VERSION_STRING, VERSION_STRING
1818highlight_language = 'none'
docs/dev/reST.rst+1−1
@@ -285,7 +285,7 @@ content becomes smart.
285285 :rst:role:`pep` :pep:`8` ``:pep:`8```
286286 sphinx.ext.extlinks_
287287 --------------------------------------------------------------------------------------------------
288− project's wiki article :wiki:`Searx-instances` ``:wiki:`Searx-instances```
288+ project's wiki article :wiki:`Offline-engines` ``:wiki:`Offline-engines```
289289 to docs public URL :docs:`dev/reST.html` ``:docs:`dev/reST.html```
290290 files & folders origin :origin:`docs/dev/reST.rst` ``:origin:`docs/dev/reST.rst```
291291 pull request :pull:`1756` ``:pull:`1756```
docs/index.rst+4−2
@@ -8,8 +8,8 @@ Searx is a free internet metasearch engine which aggregates results from more
88than 70 search services. Users are neither tracked nor profiled. Additionally,
99searx can be used over Tor for online anonymity.
1010
11−Get started with searx by using one of the :wiki:`Searx-instances`. If you
12−don't trust anyone, you can set up your own, see :ref:`installation`.
11+Get started with searx by using one of the Searx-instances_. If you don't trust
12+anyone, you can set up your own, see :ref:`installation`.
1313
1414.. sidebar:: Features
1515
@@ -32,3 +32,5 @@ don't trust anyone, you can set up your own, see :ref:`installation`.
3232 dev/index
3333 utils/index
3434 blog/index
35+
36+.. _Searx-instances: https://searx.space
searx/engines/etools.pynew file+54
@@ -0,0 +1,54 @@
1+"""
2+ eTools (Web)
3+
4+ @website https://www.etools.ch
5+ @provide-api no
6+ @using-api no
7+ @results HTML
8+ @stable no (HTML can change)
9+ @parse url, title, content
10+"""
11+
12+from lxml import html
13+from searx.engines.xpath import extract_text
14+from searx.url_utils import quote
15+from searx.utils import eval_xpath
16+
17+categories = ['general']
18+paging = False
19+language_support = False
20+safesearch = True
21+
22+base_url = 'https://www.etools.ch'
23+search_path = '/searchAdvancedSubmit.do'\
24+ '?query={search_term}'\
25+ '&pageResults=20'\
26+ '&safeSearch={safesearch}'
27+
28+
29+def request(query, params):
30+ if params['safesearch']:
31+ safesearch = 'true'
32+ else:
33+ safesearch = 'false'
34+
35+ params['url'] = base_url + search_path.format(search_term=quote(query), safesearch=safesearch)
36+
37+ return params
38+
39+
40+def response(resp):
41+ results = []
42+
43+ dom = html.fromstring(resp.text)
44+
45+ for result in eval_xpath(dom, '//table[@class="result"]//td[@class="record"]'):
46+ url = eval_xpath(result, './a/@href')[0]
47+ title = extract_text(eval_xpath(result, './a//text()'))
48+ content = extract_text(eval_xpath(result, './/div[@class="text"]//text()'))
49+
50+ results.append({'url': url,
51+ 'title': title,
52+ 'content': content})
53+
54+ return results
searx/preferences.py+7−2
@@ -4,6 +4,7 @@ from sys import version
44
55from searx import settings, autocomplete
66from searx.languages import language_codes as languages
7+from searx.utils import match_language
78from searx.url_utils import parse_qs, urlencode
89
910if version[0] == '3':
@@ -11,7 +12,7 @@ if version[0] == '3':
1112
1213
1314COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years
14−LANGUAGE_CODES = [l[0].split('-')[0] for l in languages]
15+LANGUAGE_CODES = [l[0] for l in languages]
1516LANGUAGE_CODES.append('all')
1617DISABLED = 0
1718ENABLED = 1
@@ -132,6 +133,10 @@ class SetSetting(Setting):
132133class SearchLanguageSetting(EnumStringSetting):
133134 """Available choices may change, so user's value may not be in choices anymore"""
134135
136+ def _validate_selection(self, selection):
137+ if not match_language(selection, self.choices, fallback=None) and selection != "":
138+ raise ValidationException('Invalid language code: "{0}"'.format(selection))
139+
135140 def parse(self, data):
136141 if data not in self.choices and data != self.value:
137142 # hack to give some backwards compatibility with old language cookies
@@ -268,7 +273,7 @@ class Preferences(object):
268273 super(Preferences, self).__init__()
269274
270275 self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories + ['none']),
271− 'language': SearchLanguageSetting(settings['ui']['default_locale'],
276+ 'language': SearchLanguageSetting(settings['search']['default_lang'],
272277 choices=list(LANGUAGE_CODES) + ['']),
273278 'locale': EnumStringSetting(settings['ui']['default_locale'],
274279 choices=list(settings['locales'].keys()) + ['']),
searx/settings.yml+6
@@ -5,6 +5,7 @@ general:
55search:
66 safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict
77 autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default
8+ default_lang : "" # Default search language - leave blank to detect from browser information or use codes from 'languages.py'
89 ban_time_on_fail : 5 # ban time in seconds after engine errors
910 max_ban_time_on_fail : 120 # max ban time in seconds after engine errors
1011
@@ -202,6 +203,11 @@ engines:
202203 timeout: 3.0
203204 disabled : True
204205
206+ - name : etools
207+ engine : etools
208+ shortcut : eto
209+ disabled : True
210+
205211 - name : etymonline
206212 engine : xpath
207213 paging : True
searx/webapp.py+11−11
@@ -157,14 +157,16 @@ _category_names = (gettext('files'),
157157outgoing_proxies = settings['outgoing'].get('proxies') or None
158158
159159
160+def _get_browser_language(request, lang_list):
161+ for lang in request.headers.get("Accept-Language", "en").split(","):
162+ locale = match_language(lang, lang_list, fallback=None)
163+ if locale is not None:
164+ return locale
165+
166+
160167@babel.localeselector
161168def get_locale():
162− locale = "en-US"
163−
164− for lang in request.headers.get("Accept-Language", locale).split(","):
165− locale = match_language(lang, settings['locales'].keys(), fallback=None)
166− if locale is not None:
167− break
169+ locale = _get_browser_language(request, settings['locales'].keys())
168170
169171 logger.debug("default locale from browser info is `%s`", locale)
170172
@@ -372,8 +374,7 @@ def render(template_name, override_theme=None, **kwargs):
372374 kwargs['language_codes'] = languages
373375 if 'current_language' not in kwargs:
374376 kwargs['current_language'] = match_language(request.preferences.get_value('language'),
375− LANGUAGE_CODES,
376− fallback=locale)
377+ LANGUAGE_CODES)
377378
378379 # override url_for function in templates
379380 kwargs['url_for'] = url_for_theme
@@ -444,11 +445,10 @@ def pre_request():
444445 request.errors.append(gettext('Invalid settings'))
445446
446447 # init search language and locale
447− locale = get_locale()
448448 if not preferences.get_value("language"):
449− preferences.parse_dict({"language": locale})
449+ preferences.parse_dict({"language": _get_browser_language(request, LANGUAGE_CODES)})
450450 if not preferences.get_value("locale"):
451− preferences.parse_dict({"locale": locale})
451+ preferences.parse_dict({"locale": get_locale()})
452452
453453 # request.user_plugins
454454 request.user_plugins = []