554a21e · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[enh] Add Server-Timing header (#1637)

554a21eAlexandre Flament7 years ago

searx/results.py+11

compare with master

unifiedsplit

@@ -136,6 +136,7 @@ class ResultContainer(object):

136 self._ordered = False

137 self.paging = False

138 self.unresponsive_engines = set()

139+ self.timings = []

139

140

140 def extend(self, engine_name, results):

141 def extend(self, engine_name, results):

141 for result in list(results):

142 for result in list(results):

@@ -319,3 +320,13 @@ class ResultContainer(object):

319

320

320 def add_unresponsive_engine(self, engine_error):

321 def add_unresponsive_engine(self, engine_error):

321 self.unresponsive_engines.add(engine_error)

322 self.unresponsive_engines.add(engine_error)

323+

324+ def add_timing(self, engine_name, engine_time, page_load_time):

325+ self.timings.append({

326+ 'engine': engines[engine_name].shortcut,

327+ 'total': engine_time,

328+ 'load': page_load_time

329+ })

330+

331+ def get_timings(self):

332+ return self.timings

searx/search.py+24−15

@@ -74,10 +74,10 @@ def search_one_request(engine, query, request_params):

75 # ignoring empty urls

76 if request_params['url'] is None:

77− return []

77+ return None

79 if not request_params['url']:

80− return []

80+ return None

82 # send request

83 response = send_http_request(engine, request_params)

@@ -103,20 +103,29 @@ def search_one_request_safe(engine_name, query, request_params, result_container

103 # send requests and parse the results

104 search_results = search_one_request(engine, query, request_params)

105

106+ # check if the engine accepted the request

107+ if search_results is not None:

106− # add results

108+ # yes, so add results

107− result_container.extend(engine_name, search_results)

109+ result_container.extend(engine_name, search_results)

108−

110+

109− # update engine time when there is no exception

111+ # update engine time when there is no exception

110− with threading.RLock():

111− engine.stats['engine_time'] += time() - start_time

112+ engine_time = time() - start_time

113+ page_load_time = requests_lib.get_time_for_thread()

114+ result_container.add_timing(engine_name, engine_time, page_load_time)

115+ with threading.RLock():

116+ engine.stats['engine_time'] += engine_time

112− engine.stats['engine_time_count'] += 1

117+ engine.stats['engine_time_count'] += 1

113− # update stats with the total HTTP time

118+ # update stats with the total HTTP time

114− engine.stats['page_load_time'] += requests_lib.get_time_for_thread()

119+ engine.stats['page_load_time'] += page_load_time

115− engine.stats['page_load_count'] += 1

120+ engine.stats['page_load_count'] += 1

116

121

117 except Exception as e:

122 except Exception as e:

123+ # Timing

118− search_duration = time() - start_time

124+ engine_time = time() - start_time

125+ page_load_time = requests_lib.get_time_for_thread()

126+ result_container.add_timing(engine_name, engine_time, page_load_time)

119

127

128+ # Record the errors

120 with threading.RLock():

129 with threading.RLock():

121 engine.stats['errors'] += 1

130 engine.stats['errors'] += 1

122

131

@@ -125,14 +134,14 @@ def search_one_request_safe(engine_name, query, request_params, result_container

125 # requests timeout (connect or read)

134 # requests timeout (connect or read)

126 logger.error("engine {0} : HTTP requests timeout"

135 logger.error("engine {0} : HTTP requests timeout"

127 "(search duration : {1} s, timeout: {2} s) : {3}"

136 "(search duration : {1} s, timeout: {2} s) : {3}"

128− .format(engine_name, search_duration, timeout_limit, e.__class__.__name__))

137+ .format(engine_name, engine_time, timeout_limit, e.__class__.__name__))

129 requests_exception = True

138 requests_exception = True

130 elif (issubclass(e.__class__, requests.exceptions.RequestException)):

139 elif (issubclass(e.__class__, requests.exceptions.RequestException)):

131 result_container.add_unresponsive_engine((engine_name, gettext('request exception')))

140 result_container.add_unresponsive_engine((engine_name, gettext('request exception')))

132 # other requests exception

141 # other requests exception

133 logger.exception("engine {0} : requests exception"

142 logger.exception("engine {0} : requests exception"

134 "(search duration : {1} s, timeout: {2} s) : {3}"

143 "(search duration : {1} s, timeout: {2} s) : {3}"

135− .format(engine_name, search_duration, timeout_limit, e))

144+ .format(engine_name, engine_time, timeout_limit, e))

136 requests_exception = True

145 requests_exception = True

137 else:

146 else:

138 result_container.add_unresponsive_engine((

147 result_container.add_unresponsive_engine((

searx/webapp.py+21

@@ -43,6 +43,7 @@ except:

43 exit(1)

44from cgi import escape

45from datetime import datetime, timedelta

46+from time import time

46from werkzeug.contrib.fixers import ProxyFix

47from werkzeug.contrib.fixers import ProxyFix

47from flask import (

48from flask import (

48 Flask, request, render_template, url_for, Response, make_response,

49 Flask, request, render_template, url_for, Response, make_response,

@@ -402,6 +403,8 @@ def render(template_name, override_theme=None, **kwargs):

402

403

403@app.before_request

404@app.before_request

404def pre_request():

405def pre_request():

406+ request.start_time = time()

407+ request.timings = []

405 request.errors = []

408 request.errors = []

406

409

407 preferences = Preferences(themes, list(categories.keys()), engines, plugins)

410 preferences = Preferences(themes, list(categories.keys()), engines, plugins)

@@ -437,6 +440,21 @@ def pre_request():

437 request.user_plugins.append(plugin)

440 request.user_plugins.append(plugin)

438

441

439

442

443+@app.after_request

444+def post_request(response):

445+ total_time = time() - request.start_time

446+ timings_all = ['total;dur=' + str(round(total_time * 1000, 3))]

447+ if len(request.timings) > 0:

448+ timings = sorted(request.timings, key=lambda v: v['total'])

449+ timings_total = ['total_' + str(i) + '_' + v['engine'] +

450+ ';dur=' + str(round(v['total'] * 1000, 3)) for i, v in enumerate(timings)]

451+ timings_load = ['load_' + str(i) + '_' + v['engine'] +

452+ ';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings)]

453+ timings_all = timings_all + timings_total + timings_load

454+ response.headers.add('Server-Timing', ', '.join(timings_all))

455+ return response

456+

457+

440def index_error(output_format, error_message):

458def index_error(output_format, error_message):

441 if output_format == 'json':

459 if output_format == 'json':

442 return Response(json.dumps({'error': error_message}),

460 return Response(json.dumps({'error': error_message}),

@@ -515,6 +533,9 @@ def index():

515 # UI

533 # UI

516 advanced_search = request.form.get('advanced_search', None)

534 advanced_search = request.form.get('advanced_search', None)

517

535

536+ # Server-Timing header

537+ request.timings = result_container.get_timings()

538+

518 # output

539 # output

519 for result in results:

540 for result in results:

520 if output_format == 'html':

541 if output_format == 'html':

tests/unit/test_webapp.py+15−1

@@ -33,6 +33,19 @@ class ViewsTestCase(SearxTestCase):

33 },

34 ]

36+ timings = [

37+ {

38+ 'engine': 'startpage',

39+ 'total': 0.8,

40+ 'load': 0.7

41+ },

42+ {

43+ 'engine': 'youtube',

44+ 'total': 0.9,

45+ 'load': 0.6

46+ }

47+ ]

48+

36 def search_mock(search_self, *args):

49 def search_mock(search_self, *args):

37 search_self.result_container = Mock(get_ordered_results=lambda: self.test_results,

50 search_self.result_container = Mock(get_ordered_results=lambda: self.test_results,

38 answers=set(),

51 answers=set(),

@@ -42,7 +55,8 @@ class ViewsTestCase(SearxTestCase):

42 unresponsive_engines=set(),

55 unresponsive_engines=set(),

43 results=self.test_results,

56 results=self.test_results,

44 results_number=lambda: 3,

57 results_number=lambda: 3,

45− results_length=lambda: len(self.test_results))

58+ results_length=lambda: len(self.test_results),

59+ get_timings=lambda: timings)

47 Search.search = search_mock

61 Search.search = search_mock