git.iko.soy/searx
on master help sign in
[enh] use one single http connection pool : improve response time. close #100
d07cfd9dalf
searx/autocomplete.py+1−1
compare with master
unifiedsplit
@@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
17
17
18
18
19from lxml import etree
19from lxml import etree
20−from requests import get
21from json import loads
20from json import loads
22from urllib import urlencode
21from urllib import urlencode
23from searx.languages import language_codes
22from searx.languages import language_codes
24from searx.engines import (
23from searx.engines import (
25 categories, engines, engine_shortcuts
24 categories, engines, engine_shortcuts
26)
25)
26+from searx.poolrequests import get
27
27
28
28
29def searx_bang(full_query):
29def searx_bang(full_query):
searx/engines/wikidata.py+1−1
@@ -1,6 +1,6 @@
1import json
1import json
2−from requests import get
3from urllib import urlencode
2from urllib import urlencode
3+from searx.poolrequests import get
4from searx.utils import format_date_by_locale
4from searx.utils import format_date_by_locale
5
5
6result_count = 1
6result_count = 1
searx/poolrequests.pynew file+61
@@ -0,0 +1,61 @@
1+import requests
2+
3+
4+the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
5+the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
6+
7+
8+class SessionSinglePool(requests.Session):
9+
10+ def __init__(self):
11+ global the_https_adapter, the_http_adapter
12+ super(SessionSinglePool, self).__init__()
13+
14+ # reuse the same adapters
15+ self.adapters.clear()
16+ self.mount('https://', the_https_adapter)
17+ self.mount('http://', the_http_adapter)
18+
19+ def close(self):
20+ """Call super, but clear adapters since there are managed globaly"""
21+ self.adapters.clear()
22+ super(SessionSinglePool, self).close()
23+
24+
25+def request(method, url, **kwargs):
26+ """same as requests/requests/api.py request(...) except it use SessionSinglePool"""
27+ session = SessionSinglePool()
28+ response = session.request(method=method, url=url, **kwargs)
29+ session.close()
30+ return response
31+
32+
33+def get(url, **kwargs):
34+ kwargs.setdefault('allow_redirects', True)
35+ return request('get', url, **kwargs)
36+
37+
38+def options(url, **kwargs):
39+ kwargs.setdefault('allow_redirects', True)
40+ return request('options', url, **kwargs)
41+
42+
43+def head(url, **kwargs):
44+ kwargs.setdefault('allow_redirects', False)
45+ return request('head', url, **kwargs)
46+
47+
48+def post(url, data=None, json=None, **kwargs):
49+ return request('post', url, data=data, json=json, **kwargs)
50+
51+
52+def put(url, data=None, **kwargs):
53+ return request('put', url, data=data, **kwargs)
54+
55+
56+def patch(url, data=None, **kwargs):
57+ return request('patch', url, data=data, **kwargs)
58+
59+
60+def delete(url, **kwargs):
61+ return request('delete', url, **kwargs)
searx/search.py+1−2
@@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
15(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
15(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
16'''
16'''
17
17
18−import requests as requests_lib
19import threading
18import threading
20import re
19import re
20+import searx.poolrequests as requests_lib
21from itertools import izip_longest, chain
21from itertools import izip_longest, chain
22from operator import itemgetter
22from operator import itemgetter
23from Queue import Queue
23from Queue import Queue
@@ -31,7 +31,6 @@ from searx.utils import gen_useragent
31from searx.query import Query
31from searx.query import Query
32from searx import logger
32from searx import logger
33
33
34−
35logger = logger.getChild('search')
34logger = logger.getChild('search')
36
35
37number_of_searches = 0
36number_of_searches = 0
searx/webapp.py+1−1
@@ -27,7 +27,6 @@ import cStringIO
27import os
27import os
28
28
29from datetime import datetime, timedelta
29from datetime import datetime, timedelta
30−from requests import get as http_get
31from itertools import chain
30from itertools import chain
32from urllib import urlencode
31from urllib import urlencode
33from flask import (
32from flask import (
@@ -36,6 +35,7 @@ from flask import (
36)
35)
37from flask.ext.babel import Babel, gettext, format_date
36from flask.ext.babel import Babel, gettext, format_date
38from searx import settings, searx_dir
37from searx import settings, searx_dir
38+from searx.poolrequests import get as http_get
39from searx.engines import (
39from searx.engines import (
40 categories, engines, get_engines_stats, engine_shortcuts
40 categories, engines, get_engines_stats, engine_shortcuts
41)
41)