on master help sign in
Merge pull request #2255 from kvch/yacy-improvements

Add yacy improvements: HTTP digest auth, category checking

commit835d16cbb1f896859f1757c1d952974520ec88a8 authorAdam Tauber <asciimoo@gmail.com> dateFri 9 Oct 2020, 16:34 (6 years ago) parentdafae2d72c7fd2
2 files changed+12−4
searx/engines/yacy.py+8−2 searx/search.py+4−2
searx/engines/yacy.py+8−2
@@ -16,6 +16,8 @@ from json import loads
1616from dateutil import parser
1717from urllib.parse import urlencode
1818
19+from requests.auth import HTTPDigestAuth
20+
1921from searx.utils import html_to_text
2022
2123# engine dependent config
@@ -23,6 +25,8 @@ categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'
2325paging = True
2426language_support = True
2527number_of_results = 5
28+http_digest_auth_user = ""
29+http_digest_auth_pass = ""
2630
2731# search-url
2832base_url = 'http://localhost:8090'
@@ -51,6 +55,9 @@ def request(query, params):
5155 limit=number_of_results,
5256 search_type=search_type)
5357
58+ if http_digest_auth_user and http_digest_auth_pass:
59+ params['auth'] = HTTPDigestAuth(http_digest_auth_user, http_digest_auth_pass)
60+
5461 # add language tag if specified
5562 if params['language'] != 'all':
5663 params['url'] += '&lr=lang_' + params['language'].split('-')[0]
@@ -75,7 +82,7 @@ def response(resp):
7582
7683 for result in search_results[0].get('items', []):
7784 # parse image results
78− if result.get('image') and result.get('width') and result.get('height'):
85+ if resp.search_params.get('category') == 'images':
7986
8087 result_url = ''
8188 if 'url' in result:
@@ -104,5 +111,4 @@ def response(resp):
104111
105112 # TODO parse video, audio and file results
106113
107− # return results
108114 return results
searx/search.py+4−2
@@ -97,7 +97,8 @@ def send_http_request(engine, request_params):
9797 request_args = dict(
9898 headers=request_params['headers'],
9999 cookies=request_params['cookies'],
100− verify=request_params['verify']
100+ verify=request_params['verify'],
101+ auth=request_params['auth']
101102 )
102103
103104 # setting engine based proxies
@@ -281,7 +282,8 @@ def default_request_params():
281282 'data': {},
282283 'url': '',
283284 'cookies': {},
284− 'verify': True
285+ 'verify': True,
286+ 'auth': None
285287 }
286288
287289