git.iko.soy/searx
on master help sign in
[mod] persistent multirequest storage
4f753fbAdam Tauber
searx/curladapter.py+21−16
compare with master
unifiedsplit
@@ -1,3 +1,4 @@
1+import logging
1import pycurl
2import pycurl
2from StringIO import StringIO
3from StringIO import StringIO
3from urllib import urlencode
4from urllib import urlencode
@@ -11,8 +12,8 @@ def get_new_connection(source_address=None):
11 # pycurl initialization
12 # pycurl initialization
12 h = pycurl.Curl()
13 h = pycurl.Curl()
13
14
14− # Do not follow redirect.
15+ # follow redirects
15− h.setopt(h.FOLLOWLOCATION, False)
16+ h.setopt(h.FOLLOWLOCATION, True)
16
17
17 # consistently use ipv4
18 # consistently use ipv4
18 h.setopt(h.IPRESOLVE, pycurl.IPRESOLVE_V4)
19 h.setopt(h.IPRESOLVE, pycurl.IPRESOLVE_V4)
@@ -54,6 +55,7 @@ class ConnectionCache(object):
54
55
55
56
56CONNECTION_CACHE = ConnectionCache()
57CONNECTION_CACHE = ConnectionCache()
58+MULTI_HANDLER = pycurl.CurlMulti()
57
59
58
60
59class RequestContainer(object):
61class RequestContainer(object):
@@ -87,11 +89,12 @@ class RequestContainer(object):
87 self._response_buffer = StringIO()
89 self._response_buffer = StringIO()
88 self.response = None
90 self.response = None
89 curl_handler.setopt(curl_handler.WRITEFUNCTION, self._response_buffer.write)
91 curl_handler.setopt(curl_handler.WRITEFUNCTION, self._response_buffer.write)
92+ curl_handler.setopt(curl_handler.SSL_VERIFYPEER, int(ssl_verification))
90
93
91 def _extract_response(self):
94 def _extract_response(self):
92 body = self._response_buffer.getvalue()
95 body = self._response_buffer.getvalue()
93− status = self.curl_handler.getinfo(pycurl.HTTP_CODE)
96+ status_code = self.curl_handler.getinfo(pycurl.HTTP_CODE)
94− return ResponseContainer(body, status, self.url)
97+ return ResponseContainer(body, status_code, self.url)
95
98
96 def finish(self):
99 def finish(self):
97 self.response = self._extract_response()
100 self.response = self._extract_response()
@@ -100,26 +103,33 @@ class RequestContainer(object):
100
103
101
104
102class ResponseContainer(object):
105class ResponseContainer(object):
103− def __init__(self, body, status, url):
106+ def __init__(self, body, status_code, url):
104 self.text = self.content = body
107 self.text = self.content = body
105− self.status = status
108+ self.status_code = status_code
106 self.url = url
109 self.url = url
107
110
108
111
109class MultiRequest(object):
112class MultiRequest(object):
110− def __init__(self, connection_cache=None):
113+ def __init__(self, connection_cache=None, multi_handler=None):
111 self.requests = {}
114 self.requests = {}
112− self._curl_multi_handler = pycurl.CurlMulti()
113
115
114 if connection_cache:
116 if connection_cache:
115 self.connection_cache = connection_cache
117 self.connection_cache = connection_cache
116 else:
118 else:
117 self.connection_cache = CONNECTION_CACHE
119 self.connection_cache = CONNECTION_CACHE
118
120
121+ if multi_handler:
122+ self._curl_multi_handler = multi_handler
123+ else:
124+ self._curl_multi_handler = MULTI_HANDLER
125+
119 def add(self, connection_name, url, **kwargs):
126 def add(self, connection_name, url, **kwargs):
120 handle = self.connection_cache.get_connection(connection_name)
127 handle = self.connection_cache.get_connection(connection_name)
121 request_container = RequestContainer(url, handle, **kwargs)
128 request_container = RequestContainer(url, handle, **kwargs)
129+ try:
122− self._curl_multi_handler.add_handle(handle)
130+ self._curl_multi_handler.add_handle(handle)
131+ except:
132+ pass
123 self.requests[handle] = request_container
133 self.requests[handle] = request_container
124
134
125 def perform_requests(self):
135 def perform_requests(self):
@@ -155,17 +165,12 @@ class MultiRequest(object):
155 # calling callbacks
165 # calling callbacks
156 for h in success_list:
166 for h in success_list:
157 self.requests[h].finish()
167 self.requests[h].finish()
168+ for h, err_code, err_string in error_list:
169+ logging.warn('Error on %s: "%s"', self.requests[h].url, err_string)
158 handles_num -= len(success_list) + len(error_list)
170 handles_num -= len(success_list) + len(error_list)
159 if ret != pycurl.E_CALL_MULTI_PERFORM:
171 if ret != pycurl.E_CALL_MULTI_PERFORM:
160 break
172 break
161
173
162− # response arrived
163− for k in self.requests:
164− if k.errstr():
165− print k.errstr()
166− # TODO handle errors/timeouts
167− pass
168− self._curl_multi_handler.remove_handle(k)
169 # self._curl_multi_handler.close()
174 # self._curl_multi_handler.close()
170 return self.requests.values()
175 return self.requests.values()
171
176