git.iko.soy/searx
on master help sign in
[fix] connection reuse ++ shared dns
34afcf2Adam Tauber
searx/curladapter.py+37−50
compare with master
unifiedsplit
@@ -1,14 +1,22 @@
1import logging
1import logging
2import pycurl
2import pycurl
3+import threading
4+from itertools import cycle
3from StringIO import StringIO
5from StringIO import StringIO
6+from time import time
4from urllib import urlencode
7from urllib import urlencode
5
8
6
9
10+CURL_SHARE = pycurl.CurlShare()
11+CURL_SHARE.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
12+MULTI_HANDLER = pycurl.CurlMulti()
13+
14+
7def __test_callback(*args):
15def __test_callback(*args):
8 print "callback called"
16 print "callback called"
9
17
10
18
11−def get_new_connection(source_address=None):
19+def get_connection(source_address=None):
12 # pycurl initialization
20 # pycurl initialization
13 h = pycurl.Curl()
21 h = pycurl.Curl()
14
22
@@ -18,46 +26,14 @@ def get_new_connection(source_address=None):
18 # consistently use ipv4
26 # consistently use ipv4
19 h.setopt(h.IPRESOLVE, pycurl.IPRESOLVE_V4)
27 h.setopt(h.IPRESOLVE, pycurl.IPRESOLVE_V4)
20
28
29+ h.setopt(pycurl.SHARE, CURL_SHARE)
30+
21 if source_address:
31 if source_address:
22 h.setopt(h.INTERFACE, source_address)
32 h.setopt(h.INTERFACE, source_address)
23
33
24 return h
34 return h
25
35
26
36
27−class ConnectionSources(object):
28− def __init__(self, sources=None):
29− self.sources = []
30− if sources:
31− for s in sources:
32− self.sources.append(get_new_connection(s))
33− else:
34− self.sources.append(get_new_connection())
35− self.ptr = 0
36−
37− def get_source(self):
38− source = self.sources[self.ptr]
39− self.ptr = (self.ptr + 1) % len(self.sources)
40− return source
41−
42−
43−class ConnectionCache(object):
44− def __init__(self, source_ips=None):
45− self.source_ips = source_ips if source_ips else None
46− self.connections = {}
47−
48− def get_connection(self, key):
49− if key in self.connections:
50− return self.connections[key].get_source()
51−
52− sources = ConnectionSources(self.source_ips)
53− self.connections[key] = sources
54− return sources.get_source()
55−
56−
57−CONNECTION_CACHE = ConnectionCache()
58−MULTI_HANDLER = pycurl.CurlMulti()
59−
60−
61class RequestContainer(object):
37class RequestContainer(object):
62 def __init__(self,
38 def __init__(self,
63 url,
39 url,
@@ -110,29 +86,27 @@ class ResponseContainer(object):
110
86
111
87
112class MultiRequest(object):
88class MultiRequest(object):
113− def __init__(self, connection_cache=None, multi_handler=None):
89+ def __init__(self, multi_handler=None, source_ips=None):
114 self.requests = {}
90 self.requests = {}
115
91
116− if connection_cache:
117− self.connection_cache = connection_cache
118− else:
119− self.connection_cache = CONNECTION_CACHE
120−
121 if multi_handler:
92 if multi_handler:
122 self._curl_multi_handler = multi_handler
93 self._curl_multi_handler = multi_handler
123 else:
94 else:
124 self._curl_multi_handler = MULTI_HANDLER
95 self._curl_multi_handler = MULTI_HANDLER
125
96
97+ self.source_ips = cycle(source_ips) if source_ips else cycle([None])
98+
126− def add(self, connection_name, url, **kwargs):
99+ def add(self, url, **kwargs):
127− handle = self.connection_cache.get_connection(connection_name)
100+ handle = get_connection(next(self.source_ips))
128 request_container = RequestContainer(url, handle, **kwargs)
101 request_container = RequestContainer(url, handle, **kwargs)
129 try:
102 try:
130 self._curl_multi_handler.add_handle(handle)
103 self._curl_multi_handler.add_handle(handle)
131 except:
104 except:
105+ print 'meep'
132 pass
106 pass
133 self.requests[handle] = request_container
107 self.requests[handle] = request_container
134
108
135− def perform_requests(self):
109+ def send_requests(self):
136 select_timeout = 0.1
110 select_timeout = 0.1
137
111
138 # set timeout
112 # set timeout
@@ -154,6 +128,7 @@ class MultiRequest(object):
154 else:
128 else:
155 h.unsetopt(h.COOKIE)
129 h.unsetopt(h.COOKIE)
156
130
131+ search_start = time()
157 handles_num = len(self.requests)
132 handles_num = len(self.requests)
158 while handles_num:
133 while handles_num:
159 self._curl_multi_handler.select(select_timeout)
134 self._curl_multi_handler.select(select_timeout)
@@ -164,23 +139,35 @@ class MultiRequest(object):
164 _, success_list, error_list = self._curl_multi_handler.info_read()
139 _, success_list, error_list = self._curl_multi_handler.info_read()
165 # calling callbacks
140 # calling callbacks
166 for h in success_list:
141 for h in success_list:
142+ th = threading.Thread(
143+ target=self.requests[h].finish(),
144+ name='search_request',
145+ )
146+ th.start()
167− self.requests[h].finish()
147+ # self.requests[h].finish()
168 for h, err_code, err_string in error_list:
148 for h, err_code, err_string in error_list:
169 logging.warn('Error on %s: "%s"', self.requests[h].url, err_string)
149 logging.warn('Error on %s: "%s"', self.requests[h].url, err_string)
170 handles_num -= len(success_list) + len(error_list)
150 handles_num -= len(success_list) + len(error_list)
171 if ret != pycurl.E_CALL_MULTI_PERFORM:
151 if ret != pycurl.E_CALL_MULTI_PERFORM:
172 break
152 break
173
153
154+ for th in threading.enumerate():
155+ if th.name == 'search_request':
156+ remaining_time = max(0.0, timeout - (time() - search_start))
157+ th.join(remaining_time)
158+ if th.isAlive():
159+ logging.warning('engine timeout: {0}'.format(th._engine_name))
160+
174 # self._curl_multi_handler.close()
161 # self._curl_multi_handler.close()
175 return self.requests.values()
162 return self.requests.values()
176
163
177
164
178if __name__ == '__main__':
165if __name__ == '__main__':
179 r = MultiRequest()
166 r = MultiRequest()
180− r.add('a', 'http://httpbin.org/delay/0', headers={'User-Agent': 'x'})
167+ r.add('http://httpbin.org/delay/0', headers={'User-Agent': 'x'})
181− r.add('d', 'http://127.0.0.1:7777/', headers={'User-Agent': 'x'})
168+ r.add('http://127.0.0.1:7777/', headers={'User-Agent': 'x'})
182− r.add('b', 'http://httpbin.org/delay/0', cookies={'as': 'sa', 'bb': 'cc'})
169+ r.add('http://httpbin.org/delay/0', cookies={'as': 'sa', 'bb': 'cc'})
183− r.add('c', 'http://httpbin.org/delay/0', callback=__test_callback, timeout=1.0, headers={'User-Agent': 'x'})
170+ r.add('http://httpbin.org/delay/0', callback=__test_callback, timeout=1.0, headers={'User-Agent': 'x'})
184− for v in r.perform_requests():
171+ for v in r.send_requests():
185 print v.url
172 print v.url
186 print v.response.text
173 print v.response.text
searx/search.py+2−4
@@ -320,15 +320,13 @@ class Search(object):
320 continue
320 continue
321
321
322 # append request to list
322 # append request to list
323− mr.add((selected_engine['name'], selected_engine['category']),
324− request_params['url'],
323+ mr.add(request_params['url'], **request_args)
325− **request_args)
326
324
327 if not mr.requests:
325 if not mr.requests:
328 return self
326 return self
329
327
330 # send all search-request
328 # send all search-request
331− mr.perform_requests()
329+ mr.send_requests()
332
330
333 # return results, suggestions, answers and infoboxes
331 # return results, suggestions, answers and infoboxes
334 return self
332 return self