on master help sign in
[enh] use one single http connection pool : improve response time. close #100
commitd07cfd9089d05a6a81109518f03ba82660a4aef7 authordalf <alex@al-f.net> dateWed 21 Jan 2015, 11:33 (12 years ago) parenta865e66
5 files changed+65−5
searx/autocomplete.py+1−1 searx/engines/wikidata.py+1−1 searx/poolrequests.py+61 searx/search.py+1−2 searx/webapp.py+1−1
searx/autocomplete.py+1−1
@@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
1717
1818
1919from lxml import etree
20−from requests import get
2120from json import loads
2221from urllib import urlencode
2322from searx.languages import language_codes
2423from searx.engines import (
2524 categories, engines, engine_shortcuts
2625)
26+from searx.poolrequests import get
2727
2828
2929def searx_bang(full_query):
searx/engines/wikidata.py+1−1
@@ -1,6 +1,6 @@
11import json
2−from requests import get
32from urllib import urlencode
3+from searx.poolrequests import get
44from searx.utils import format_date_by_locale
55
66result_count = 1
searx/poolrequests.pynew file+61
@@ -0,0 +1,61 @@
1+import requests
2+
3+
4+the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
5+the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
6+
7+
8+class SessionSinglePool(requests.Session):
9+
10+ def __init__(self):
11+ global the_https_adapter, the_http_adapter
12+ super(SessionSinglePool, self).__init__()
13+
14+ # reuse the same adapters
15+ self.adapters.clear()
16+ self.mount('https://', the_https_adapter)
17+ self.mount('http://', the_http_adapter)
18+
19+ def close(self):
20+ """Call super, but clear adapters since there are managed globaly"""
21+ self.adapters.clear()
22+ super(SessionSinglePool, self).close()
23+
24+
25+def request(method, url, **kwargs):
26+ """same as requests/requests/api.py request(...) except it use SessionSinglePool"""
27+ session = SessionSinglePool()
28+ response = session.request(method=method, url=url, **kwargs)
29+ session.close()
30+ return response
31+
32+
33+def get(url, **kwargs):
34+ kwargs.setdefault('allow_redirects', True)
35+ return request('get', url, **kwargs)
36+
37+
38+def options(url, **kwargs):
39+ kwargs.setdefault('allow_redirects', True)
40+ return request('options', url, **kwargs)
41+
42+
43+def head(url, **kwargs):
44+ kwargs.setdefault('allow_redirects', False)
45+ return request('head', url, **kwargs)
46+
47+
48+def post(url, data=None, json=None, **kwargs):
49+ return request('post', url, data=data, json=json, **kwargs)
50+
51+
52+def put(url, data=None, **kwargs):
53+ return request('put', url, data=data, **kwargs)
54+
55+
56+def patch(url, data=None, **kwargs):
57+ return request('patch', url, data=data, **kwargs)
58+
59+
60+def delete(url, **kwargs):
61+ return request('delete', url, **kwargs)
searx/search.py+1−2
@@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
1515(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
1616'''
1717
18−import requests as requests_lib
1918import threading
2019import re
20+import searx.poolrequests as requests_lib
2121from itertools import izip_longest, chain
2222from operator import itemgetter
2323from Queue import Queue
@@ -31,7 +31,6 @@ from searx.utils import gen_useragent
3131from searx.query import Query
3232from searx import logger
3333
34−
3534logger = logger.getChild('search')
3635
3736number_of_searches = 0
searx/webapp.py+1−1
@@ -27,7 +27,6 @@ import cStringIO
2727import os
2828
2929from datetime import datetime, timedelta
30−from requests import get as http_get
3130from itertools import chain
3231from urllib import urlencode
3332from flask import (
@@ -36,6 +35,7 @@ from flask import (
3635)
3736from flask.ext.babel import Babel, gettext, format_date
3837from searx import settings, searx_dir
38+from searx.poolrequests import get as http_get
3939from searx.engines import (
4040 categories, engines, get_engines_stats, engine_shortcuts
4141)