@@ -1,3 +1,4 @@
1+import logging
12import pycurl
23from StringIO import StringIO
34from urllib import urlencode
@@ -11,8 +12,8 @@ def get_new_connection(source_address=None):
1112 # pycurl initialization
1213 h = pycurl.Curl()
1314
14− # Do not follow redirect.
15− h.setopt(h.FOLLOWLOCATION, False)
15+ # follow redirects
16+ h.setopt(h.FOLLOWLOCATION, True)
1617
1718 # consistently use ipv4
1819 h.setopt(h.IPRESOLVE, pycurl.IPRESOLVE_V4)
@@ -54,6 +55,7 @@ class ConnectionCache(object):
5455
5556
5657CONNECTION_CACHE = ConnectionCache()
58+MULTI_HANDLER = pycurl.CurlMulti()
5759
5860
5961class RequestContainer(object):
@@ -87,11 +89,12 @@ class RequestContainer(object):
8789 self._response_buffer = StringIO()
8890 self.response = None
8991 curl_handler.setopt(curl_handler.WRITEFUNCTION, self._response_buffer.write)
92+ curl_handler.setopt(curl_handler.SSL_VERIFYPEER, int(ssl_verification))
9093
9194 def _extract_response(self):
9295 body = self._response_buffer.getvalue()
93− status = self.curl_handler.getinfo(pycurl.HTTP_CODE)
94− return ResponseContainer(body, status, self.url)
96+ status_code = self.curl_handler.getinfo(pycurl.HTTP_CODE)
97+ return ResponseContainer(body, status_code, self.url)
9598
9699 def finish(self):
97100 self.response = self._extract_response()
@@ -100,26 +103,33 @@ class RequestContainer(object):
100103
101104
102105class ResponseContainer(object):
103− def __init__(self, body, status, url):
106+ def __init__(self, body, status_code, url):
104107 self.text = self.content = body
105− self.status = status
108+ self.status_code = status_code
106109 self.url = url
107110
108111
109112class MultiRequest(object):
110− def __init__(self, connection_cache=None):
113+ def __init__(self, connection_cache=None, multi_handler=None):
111114 self.requests = {}
112− self._curl_multi_handler = pycurl.CurlMulti()
113115
114116 if connection_cache:
115117 self.connection_cache = connection_cache
116118 else:
117119 self.connection_cache = CONNECTION_CACHE
118120
121+ if multi_handler:
122+ self._curl_multi_handler = multi_handler
123+ else:
124+ self._curl_multi_handler = MULTI_HANDLER
125+
119126 def add(self, connection_name, url, **kwargs):
120127 handle = self.connection_cache.get_connection(connection_name)
121128 request_container = RequestContainer(url, handle, **kwargs)
122− self._curl_multi_handler.add_handle(handle)
129+ try:
130+ self._curl_multi_handler.add_handle(handle)
131+ except:
132+ pass
123133 self.requests[handle] = request_container
124134
125135 def perform_requests(self):
@@ -155,17 +165,12 @@ class MultiRequest(object):
155165 # calling callbacks
156166 for h in success_list:
157167 self.requests[h].finish()
168+ for h, err_code, err_string in error_list:
169+ logging.warn('Error on %s: "%s"', self.requests[h].url, err_string)
158170 handles_num -= len(success_list) + len(error_list)
159171 if ret != pycurl.E_CALL_MULTI_PERFORM:
160172 break
161173
162− # response arrived
163− for k in self.requests:
164− if k.errstr():
165− print k.errstr()
166− # TODO handle errors/timeouts
167− pass
168− self._curl_multi_handler.remove_handle(k)
169174 # self._curl_multi_handler.close()
170175 return self.requests.values()
171176