@@ -0,0 +1,182 @@
1+import pycurl
2+import sys
3+from StringIO import StringIO
4+from urllib import urlencode
5+
6+
7+def a(*args):
8+ sys.stdout.write('*'*10+repr(args)+'\n')
9+
10+
11+def get_new_connection(source_address=None):
12+ # pycurl initialization
13+ h = pycurl.Curl()
14+
15+ # Do not follow redirect.
16+ h.setopt(h.FOLLOWLOCATION, False)
17+
18+ # consistently use ipv4
19+ h.setopt(h.IPRESOLVE, pycurl.IPRESOLVE_V4)
20+
21+ if source_address:
22+ h.setopt(h.INTERFACE, source_address)
23+
24+ return h
25+
26+
27+class ConnectionSources(object):
28+ def __init__(self, sources=None):
29+ self.sources = []
30+ if sources:
31+ for s in sources:
32+ self.sources.append(get_new_connection(s))
33+ else:
34+ self.sources.append(get_new_connection())
35+ self.ptr = 0
36+
37+ def get_source(self):
38+ source = self.sources[self.ptr]
39+ self.ptr = (self.ptr + 1) % len(self.sources)
40+ return source
41+
42+
43+class ConnectionCache(object):
44+ def __init__(self, source_ips=None):
45+ self.source_ips = source_ips if source_ips else None
46+ self.connections = {}
47+
48+ def get_connection(self, key):
49+ if key in self.connections:
50+ return self.connections[key].get_source()
51+
52+ sources = ConnectionSources(self.source_ips)
53+ self.connections[key] = sources
54+ return sources.get_source()
55+
56+
57+CONNECTION_CACHE = ConnectionCache()
58+
59+
60+class RequestContainer(object):
61+ def __init__(self,
62+ url,
63+ curl_handler,
64+ method='GET',
65+ headers=None,
66+ cookies=None,
67+ callback=None,
68+ data=None,
69+ timeout=2.0,
70+ ssl_verification=True):
71+
72+ if headers is None:
73+ headers = {}
74+
75+ if cookies is None:
76+ cookies = {}
77+
78+ if data is not None:
79+ curl_handler.setopt(curl_handler.POSTFIELDS, urlencode(data))
80+
81+ self.url = url
82+ self.headers = headers
83+ self.cookies = cookies
84+ self.timeout = int(timeout * 1000) # in milisecs
85+ self.callback = callback
86+ self.curl_handler = curl_handler
87+
88+ self._response_buffer = StringIO()
89+ self.response = None
90+ curl_handler.setopt(curl_handler.WRITEFUNCTION, self._response_buffer.write)
91+
92+ def _extract_response(self):
93+ body = self._response_buffer.getvalue()
94+ status = self.curl_handler.getinfo(pycurl.HTTP_CODE)
95+ return ResponseContainer(body, status, self.url)
96+
97+ def finish(self):
98+ self.response = self._extract_response()
99+ if self.callback:
100+ return self.callback(self.response)
101+
102+
103+class ResponseContainer(object):
104+ def __init__(self, body, status, url):
105+ self.text = self.content = body
106+ self.status = status
107+ self.url = url
108+
109+
110+class MultiRequest(object):
111+ def __init__(self, connection_cache=None):
112+ self.requests = {}
113+ self._curl_multi_handler = pycurl.CurlMulti()
114+
115+ if connection_cache:
116+ self.connection_cache = connection_cache
117+ else:
118+ self.connection_cache = CONNECTION_CACHE
119+
120+ def add(self, connection_name, url, **kwargs):
121+ handle = self.connection_cache.get_connection(connection_name)
122+ request_container = RequestContainer(url, handle, **kwargs)
123+ self._curl_multi_handler.add_handle(handle)
124+ self.requests[handle] = request_container
125+
126+ def perform_requests(self):
127+ select_timeout = 0.1
128+
129+ # set timeout
130+ timeout = max(c.timeout for c in self.requests.values())
131+ for h, c in self.requests.iteritems():
132+ h.setopt(h.CONNECTTIMEOUT_MS, timeout)
133+ h.setopt(h.TIMEOUT_MS, timeout)
134+ h.setopt(h.URL, c.url)
135+ c.headers['Connection'] = 'keep-alive'
136+ # c.headers['Accept-Encoding'] = 'gzip, deflate'
137+
138+ h.setopt(h.HTTPHEADER,
139+ ['{0}: {1}'.format(k, v)
140+ for k, v in c.headers.iteritems()])
141+
142+ if c.cookies:
143+ h.setopt(h.COOKIE, '; '.join('{0}={1}'.format(k, v)
144+ for k, v in c.cookies.iteritems()))
145+ else:
146+ h.unsetopt(h.COOKIE)
147+
148+ handles_num = len(self.requests)
149+ while handles_num:
150+ self._curl_multi_handler.select(select_timeout)
151+ while 1:
152+ ret, new_handles_num = self._curl_multi_handler.perform()
153+ # handle finished
154+ if new_handles_num < handles_num:
155+ _, success_list, error_list = self._curl_multi_handler.info_read()
156+ # calling callbacks
157+ for h in success_list:
158+ self.requests[h].finish()
159+ handles_num -= len(success_list) + len(error_list)
160+ if ret != pycurl.E_CALL_MULTI_PERFORM:
161+ break
162+
163+ # response arrived
164+ for k in self.requests:
165+ if k.errstr():
166+ print k.errstr()
167+ # TODO handle errors/timeouts
168+ pass
169+ self._curl_multi_handler.remove_handle(k)
170+ # self._curl_multi_handler.close()
171+ return self.requests.values()
172+
173+
174+if __name__ == '__main__':
175+ r = MultiRequest()
176+ r.add('a', 'http://httpbin.org/delay/0', headers={'User-Agent': 'x'})
177+ r.add('d', 'http://127.0.0.1:7777/', headers={'User-Agent': 'x'})
178+ r.add('b', 'http://httpbin.org/delay/0', cookies={'as': 'sa', 'bb': 'cc'})
179+ r.add('c', 'http://httpbin.org/delay/0', callback=a, timeout=1.0, headers={'User-Agent': 'x'})
180+ for v in r.perform_requests():
181+ print v.url
182+ print v.response.text