c1cfe97 · searx · git.iko.soy

git.iko.soy/searx

[enh] timeout and total HTTP time are managed by searx.poolrequests

commitc1cfe978511c800cdbdbebee2306139524dc715b authorAlexandre Flament <alex@al-f.net> dateSun 23 Jul 2017, 11:56 (9 years ago) parentbf02b86

2 files changed+79−42

compare with master

unifiedsplit

searx/poolrequests.py+49−2 searx/search.py+30−40

searx/poolrequests.py+49−2

@@ -1,8 +1,9 @@

11import requests

33from itertools import cycle

4−from threading import RLock

4+from threading import RLock, local

55from searx import settings

6+from time import time

89class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):

@@ -41,6 +42,7 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):

4142 block=self._pool_block, **self._conn_params)

4243

4344

45+threadLocal = local()

4446connect = settings['outgoing'].get('pool_connections', 100) # Magic number kept from previous code

4547maxsize = settings['outgoing'].get('pool_maxsize', requests.adapters.DEFAULT_POOLSIZE) # Picked from constructor

4648if settings['outgoing'].get('source_ips'):

@@ -72,12 +74,57 @@ class SessionSinglePool(requests.Session):

7274 super(SessionSinglePool, self).close()

7375

7476

77+def set_timeout_for_thread(timeout, start_time=None):

78+ threadLocal.timeout = timeout

79+ threadLocal.start_time = start_time

80+

81+

82+def reset_time_for_thread():

83+ threadLocal.total_time = 0

84+

85+

86+def get_time_for_thread():

87+ return threadLocal.total_time

88+

89+

7590def request(method, url, **kwargs):

76− """same as requests/requests/api.py request(...) except it use SessionSinglePool and force proxies"""

91+ """same as requests/requests/api.py request(...)"""

92+ time_before_request = time()

93+

94+ # session start

7795 session = SessionSinglePool()

96+

97+ # proxies

7898 kwargs['proxies'] = settings['outgoing'].get('proxies') or None

99+

100+ # timeout

101+ if 'timeout' in kwargs:

102+ timeout = kwargs['timeout']

103+ else:

104+ timeout = getattr(threadLocal, 'timeout', None)

105+ if timeout is not None:

106+ kwargs['timeout'] = timeout

107+

108+ # do request

79109 response = session.request(method=method, url=url, **kwargs)

110+

111+ time_after_request = time()

112+

113+ # is there a timeout for this engine ?

114+ if timeout is not None:

115+ timeout_overhead = 0.2 # seconds

116+ # start_time = when the user request started

117+ start_time = getattr(threadLocal, 'start_time', time_before_request)

118+ search_duration = time_after_request - start_time

119+ if search_duration > timeout + timeout_overhead:

120+ raise requests.exceptions.Timeout(response=response)

121+

122+ # session end

80123 session.close()

124+

125+ #

126+ threadLocal.total_time += time_after_request - time_before_request

127+

81128 return response

82129

83130

searx/search.py+30−40

@@ -47,16 +47,12 @@ logger = logger.getChild('search')

4747number_of_searches = 0

4848

4949

50−def send_http_request(engine, request_params, start_time, timeout_limit):

51− # for page_load_time stats

52− time_before_request = time()

53−

50+def send_http_request(engine, request_params):

5451 # create dictionary which contain all

5552 # informations about the request

5653 request_args = dict(

5754 headers=request_params['headers'],

5855 cookies=request_params['cookies'],

59− timeout=timeout_limit,

6056 verify=request_params['verify']

6157 )

6258

@@ -68,29 +64,10 @@ def send_http_request(engine, request_params, start_time, timeout_limit):

6864 request_args['data'] = request_params['data']

6965

7066 # send the request

71− response = req(request_params['url'], **request_args)

72−

73− # is there a timeout (no parsing in this case)

74− timeout_overhead = 0.2 # seconds

75− time_after_request = time()

76− search_duration = time_after_request - start_time

77− if search_duration > timeout_limit + timeout_overhead:

78− raise requests.exceptions.Timeout(response=response)

79−

80− with threading.RLock():

81− # no error : reset the suspend variables

82− engine.continuous_errors = 0

83− engine.suspend_end_time = 0

84− # update stats with current page-load-time

85− # only the HTTP request

86− engine.stats['page_load_time'] += time_after_request - time_before_request

87− engine.stats['page_load_count'] += 1

67+ return req(request_params['url'], **request_args)

8868

89− # everything is ok : return the response

90− return response

9169

92−

93−def search_one_request(engine, query, request_params, start_time, timeout_limit):

70+def search_one_request(engine, query, request_params):

9471 # update request parameters dependent on

9572 # search-engine (contained in engines folder)

9673 engine.request(query, request_params)

@@ -103,7 +80,7 @@ def search_one_request(engine, query, request_params, start_time, timeout_limit)

10380 return []

10481

10582 # send request

106− response = send_http_request(engine, request_params, start_time, timeout_limit)

83+ response = send_http_request(engine, request_params)

10784

10885 # parse the response

10986 response.search_params = request_params

@@ -111,11 +88,20 @@ def search_one_request(engine, query, request_params, start_time, timeout_limit)

11188

11289

11390def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):

91+ # set timeout for all HTTP requests

92+ requests_lib.set_timeout_for_thread(timeout_limit, start_time=start_time)

93+ # reset the HTTP total time

94+ requests_lib.reset_time_for_thread()

95+

96+ #

11497 engine = engines[engine_name]

11598

99+ # suppose everything will be alright

100+ requests_exception = False

101+

116102 try:

117103 # send requests and parse the results

118− search_results = search_one_request(engine, query, request_params, start_time, timeout_limit)

104+ search_results = search_one_request(engine, query, request_params)

119105

120106 # add results

121107 result_container.extend(engine_name, search_results)

@@ -124,14 +110,15 @@ def search_one_request_safe(engine_name, query, request_params, result_container

124110 with threading.RLock():

125111 engine.stats['engine_time'] += time() - start_time

126112 engine.stats['engine_time_count'] += 1

127−

128− return True

113+ # update stats with the total HTTP time

114+ engine.stats['page_load_time'] += requests_lib.get_time_for_thread()

115+ engine.stats['page_load_count'] += 1

129116

130117 except Exception as e:

131− engine.stats['errors'] += 1

132−

133118 search_duration = time() - start_time

134− requests_exception = False

119+

120+ with threading.RLock():

121+ engine.stats['errors'] += 1

135122

136123 if (issubclass(e.__class__, requests.exceptions.Timeout)):

137124 result_container.add_unresponsive_engine((engine_name, gettext('timeout')))

@@ -152,14 +139,17 @@ def search_one_request_safe(engine_name, query, request_params, result_container

152139 # others errors

153140 logger.exception('engine {0} : exception : {1}'.format(engine_name, e))

154141

155− # update continuous_errors / suspend_end_time

142+ # suspend or not the engine if there are HTTP errors

143+ with threading.RLock():

156144 if requests_exception:

157− with threading.RLock():

158− engine.continuous_errors += 1

159− engine.suspend_end_time = time() + min(60, engine.continuous_errors)

160−

161− #

162− return False

145+ # update continuous_errors / suspend_end_time

146+ engine.continuous_errors += 1

147+ engine.suspend_end_time = time() + min(60, engine.continuous_errors)

148+ else:

149+ # no HTTP error (perhaps an engine error)

150+ # anyway, reset the suspend variables

151+ engine.continuous_errors = 0

152+ engine.suspend_end_time = 0

163153

164154

165155def search_multiple_requests(requests, result_container, start_time, timeout_limit):