git.iko.soy/searx
on master help sign in
Merge branch 'master' of https://github.com/asciimoo/searx into filtron
commit1fdd8cd3c0e5cb51e655baba3c1f9426e55e0add authorMarkus Heiser <markus.heiser@darmarit.de> dateWed 22 Apr 2020, 14:24 (7 years ago) parentd21e0b0ac2b0ff
6 files changed+29−128
compare with master
unifiedsplit
searx/data/useragents.json+5−5 searx/engines/faroo.py−96 searx/results.py+2−2 searx/search.py+5−12 searx/settings.yml−5 searx/webapp.py+17−8
searx/data/useragents.json+5−5
@@ -1,12 +1,12 @@
11{
22 "versions": [
3− "74.0",
4− "73.0.1",
5− "73.0"
3+ "75.0",
4+ "74.0.1",
5+ "74.0"
66 ],
77 "os": [
8− "Windows NT 10; WOW64",
8+ "Windows NT 10.0; WOW64",
99 "X11; Linux x86_64"
1010 ],
1111 "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
12−}
12+}
searx/engines/faroo.pydeleted−96
@@ -1,96 +0,0 @@
1−"""
2− Faroo (Web, News)
3−
4− @website http://www.faroo.com
5− @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key
6−
7− @using-api no
8− @results JSON
9− @stable yes
10− @parse url, title, content, publishedDate, img_src
11−"""
12−
13−from json import loads
14−import datetime
15−from searx.utils import searx_useragent
16−from searx.url_utils import urlencode
17−
18−# engine dependent config
19−categories = ['general', 'news']
20−paging = True
21−language_support = True
22−number_of_results = 10
23−
24−# search-url
25−url = 'http://www.faroo.com/'
26−search_url = url + 'instant.json?{query}'\
27− '&start={offset}'\
28− '&length={number_of_results}'\
29− '&l={language}'\
30− '&src={categorie}'\
31− '&i=false'\
32− '&c=false'
33−
34−search_category = {'general': 'web',
35− 'news': 'news'}
36−
37−
38−# do search-request
39−def request(query, params):
40− offset = (params['pageno'] - 1) * number_of_results + 1
41− categorie = search_category.get(params['category'], 'web')
42−
43− if params['language'] == 'all':
44− language = 'en'
45− else:
46− language = params['language'].split('-')[0]
47−
48− # if language is not supported, put it in english
49− if language != 'en' and\
50− language != 'de' and\
51− language != 'zh':
52− language = 'en'
53−
54− params['url'] = search_url.format(offset=offset,
55− number_of_results=number_of_results,
56− query=urlencode({'q': query}),
57− language=language,
58− categorie=categorie)
59−
60− params['headers']['Referer'] = url
61−
62− return params
63−
64−
65−# get response from search-request
66−def response(resp):
67− # HTTP-Code 429: rate limit exceeded
68− if resp.status_code == 429:
69− raise Exception("rate limit has been exceeded!")
70−
71− results = []
72−
73− search_res = loads(resp.text)
74−
75− # return empty array if there are no results
76− if not search_res.get('results', {}):
77− return []
78−
79− # parse results
80− for result in search_res['results']:
81− publishedDate = None
82− result_json = {'url': result['url'], 'title': result['title'],
83− 'content': result['kwic']}
84− if result['news']:
85− result_json['publishedDate'] = \
86− datetime.datetime.fromtimestamp(result['date'] / 1000.0)
87−
88− # append image result if image url is set
89− if result['iurl']:
90− result_json['template'] = 'videos.html'
91− result_json['thumbnail'] = result['iurl']
92−
93− results.append(result_json)
94−
95− # return results
96− return results
searx/results.py+2−2
@@ -345,8 +345,8 @@ class ResultContainer(object):
345345 return 0
346346 return resultnum_sum / len(self._number_of_results)
347347
348− def add_unresponsive_engine(self, engine_error):
349− self.unresponsive_engines.add(engine_error)
348+ def add_unresponsive_engine(self, engine_name, error_type, error_message=None):
349+ self.unresponsive_engines.add((engine_name, error_type, error_message))
350350
351351 def add_timing(self, engine_name, engine_time, page_load_time):
352352 self.timings.append({
searx/search.py+5−12
@@ -127,11 +127,7 @@ def search_one_offline_request_safe(engine_name, query, request_params, result_c
127127 logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))
128128 except Exception as e:
129129 record_offline_engine_stats_on_error(engine, result_container, start_time)
130−
131− result_container.add_unresponsive_engine((
132− engine_name,
133− u'{0}: {1}'.format(gettext('unexpected crash'), e),
134− ))
130+ result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
135131 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
136132
137133
@@ -186,24 +182,21 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont
186182 engine.stats['errors'] += 1
187183
188184 if (issubclass(e.__class__, requests.exceptions.Timeout)):
189− result_container.add_unresponsive_engine((engine_name, gettext('timeout')))
185+ result_container.add_unresponsive_engine(engine_name, 'timeout')
190186 # requests timeout (connect or read)
191187 logger.error("engine {0} : HTTP requests timeout"
192188 "(search duration : {1} s, timeout: {2} s) : {3}"
193189 .format(engine_name, engine_time, timeout_limit, e.__class__.__name__))
194190 requests_exception = True
195191 elif (issubclass(e.__class__, requests.exceptions.RequestException)):
196− result_container.add_unresponsive_engine((engine_name, gettext('request exception')))
192+ result_container.add_unresponsive_engine(engine_name, 'request exception')
197193 # other requests exception
198194 logger.exception("engine {0} : requests exception"
199195 "(search duration : {1} s, timeout: {2} s) : {3}"
200196 .format(engine_name, engine_time, timeout_limit, e))
201197 requests_exception = True
202198 else:
203− result_container.add_unresponsive_engine((
204− engine_name,
205− u'{0}: {1}'.format(gettext('unexpected crash'), e),
206− ))
199+ result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))
207200 # others errors
208201 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
209202
@@ -238,7 +231,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim
238231 remaining_time = max(0.0, timeout_limit - (time() - start_time))
239232 th.join(remaining_time)
240233 if th.isAlive():
241− result_container.add_unresponsive_engine((th._engine_name, gettext('timeout')))
234+ result_container.add_unresponsive_engine(th._engine_name, 'timeout')
242235 logger.warning('engine timeout: {0}'.format(th._engine_name))
243236
244237
searx/settings.yml−5
@@ -219,11 +219,6 @@ engines:
219219 shortcut : et
220220 disabled : True
221221
222− - name : faroo
223− engine : faroo
224− shortcut : fa
225− disabled : True
226−
227222 - name : 1x
228223 engine : www1x
229224 shortcut : 1x
searx/webapp.py+17−8
@@ -56,6 +56,7 @@ from flask import (
5656from babel.support import Translations
5757import flask_babel
5858from flask_babel import Babel, gettext, format_date, format_decimal
59+from flask.ctx import has_request_context
5960from flask.json import jsonify
6061from searx import brand
6162from searx import settings, searx_dir, searx_debug
@@ -165,13 +166,11 @@ _flask_babel_get_translations = flask_babel.get_translations
165166
166167# monkey patch for flask_babel.get_translations
167168def _get_translations():
168− translation_locale = request.form.get('use-translation')
169− if translation_locale:
169+ if has_request_context() and request.form.get('use-translation') == 'oc':
170170 babel_ext = flask_babel.current_app.extensions['babel']
171− translation = Translations.load(next(babel_ext.translation_directories), 'oc')
172− else:
173− translation = _flask_babel_get_translations()
174− return translation
171+ return Translations.load(next(babel_ext.translation_directories), 'oc')
172+
173+ return _flask_babel_get_translations()
175174
176175
177176flask_babel.get_translations = _get_translations
@@ -627,7 +626,7 @@ def index():
627626 'corrections': list(result_container.corrections),
628627 'infoboxes': result_container.infoboxes,
629628 'suggestions': list(result_container.suggestions),
630− 'unresponsive_engines': list(result_container.unresponsive_engines)},
629+ 'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)}, # noqa
631630 default=lambda item: list(item) if isinstance(item, set) else item),
632631 mimetype='application/json')
633632 elif output_format == 'csv':
@@ -695,7 +694,7 @@ def index():
695694 corrections=correction_urls,
696695 infoboxes=result_container.infoboxes,
697696 paging=result_container.paging,
698− unresponsive_engines=result_container.unresponsive_engines,
697+ unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),
699698 current_language=match_language(search_query.lang,
700699 LANGUAGE_CODES,
701700 fallback=request.preferences.get_value("language")),
@@ -706,6 +705,16 @@ def index():
706705 )
707706
708707
708+def __get_translated_errors(unresponsive_engines):
709+ translated_errors = []
710+ for unresponsive_engine in unresponsive_engines:
711+ error_msg = gettext(unresponsive_engine[1])
712+ if unresponsive_engine[2]:
713+ error_msg = "{} {}".format(error_msg, unresponsive_engine[2])
714+ translated_errors.append((unresponsive_engine[0], error_msg))
715+ return translated_errors
716+
717+
709718@app.route('/about', methods=['GET'])
710719def about():
711720 """Render about page"""