mirror of
https://github.com/ilyakooo0/searx.git
synced 2024-09-11 09:15:54 +03:00
[enh] gevent/grequests changed to the built-in threading lib
This commit is contained in:
parent
b1b0b82a2a
commit
d959cb1c05
@ -1,6 +1,6 @@
|
||||
flask
|
||||
flask-babel
|
||||
grequests
|
||||
requests
|
||||
lxml
|
||||
pyyaml
|
||||
python-dateutil
|
||||
|
@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
||||
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
||||
'''
|
||||
|
||||
import grequests
|
||||
import requests as requests_lib
|
||||
import threading
|
||||
import re
|
||||
from itertools import izip_longest, chain
|
||||
from datetime import datetime
|
||||
@ -32,6 +33,18 @@ from searx.query import Query
|
||||
number_of_searches = 0
|
||||
|
||||
|
||||
def threaded_requests(requests):
|
||||
for fn, url, request_args in requests:
|
||||
th = threading.Thread(
|
||||
target=fn, args=(url,), kwargs=request_args, name=url,
|
||||
)
|
||||
th.start()
|
||||
|
||||
for th in threading.enumerate():
|
||||
if th.name.startswith('http'):
|
||||
th.join()
|
||||
|
||||
|
||||
# get default reqest parameter
|
||||
def default_request_params():
|
||||
return {
|
||||
@ -471,9 +484,9 @@ class Search(object):
|
||||
|
||||
# specific type of request (GET or POST)
|
||||
if request_params['method'] == 'GET':
|
||||
req = grequests.get
|
||||
req = requests_lib.get
|
||||
else:
|
||||
req = grequests.post
|
||||
req = requests_lib.post
|
||||
request_args['data'] = request_params['data']
|
||||
|
||||
# ignoring empty urls
|
||||
@ -481,10 +494,10 @@ class Search(object):
|
||||
continue
|
||||
|
||||
# append request to list
|
||||
requests.append(req(request_params['url'], **request_args))
|
||||
requests.append((req, request_params['url'], request_args))
|
||||
|
||||
# send all search-request
|
||||
grequests.map(requests)
|
||||
threaded_requests(requests)
|
||||
|
||||
# update engine-specific stats
|
||||
for engine_name, engine_results in results.items():
|
||||
|
@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
|
||||
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
|
||||
'''
|
||||
|
||||
from gevent import monkey
|
||||
monkey.patch_all()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from sys import path
|
||||
from os.path import realpath, dirname
|
||||
|
2
setup.py
2
setup.py
@ -41,7 +41,7 @@ setup(
|
||||
install_requires=[
|
||||
'flask',
|
||||
'flask-babel',
|
||||
'grequests',
|
||||
'requests',
|
||||
'lxml',
|
||||
'pyyaml',
|
||||
'setuptools',
|
||||
|
12
versions.cfg
12
versions.cfg
@ -36,18 +36,6 @@ zc.recipe.testrunner = 2.0.0
|
||||
# WebTest==2.0.11
|
||||
beautifulsoup4 = 4.3.2
|
||||
|
||||
# Required by:
|
||||
# grequests==0.2.0
|
||||
gevent = 1.0
|
||||
|
||||
# Required by:
|
||||
# gevent==1.0
|
||||
greenlet = 0.4.2
|
||||
|
||||
# Required by:
|
||||
# searx==0.1
|
||||
grequests = 0.2.0
|
||||
|
||||
# Required by:
|
||||
# robotframework-httplibrary==0.4.2
|
||||
jsonpatch = 1.3
|
||||
|
Loading…
Reference in New Issue
Block a user