on master help sign in
8 files changed+117−48
since splitsnapshots
unifiedsplit
Dockerfile+3−2 searx/autocomplete.py+6−6 searx/engines/gigablast.py+24−13 searx/network/__init__.py+22−1 searx/network/network.py+8 searx/settings.yml+12−8 searx/utils.py+1−5 searx/webapp.py+41−13
Dockerfile+3−2
@@ -51,13 +51,14 @@ RUN apk upgrade --no-cache \
5151 && apk del build-dependencies \
5252 && rm -rf /root/.cache
5353
54−COPY --chown=searx:searx . .
54+COPY searx ./searx
55+COPY dockerfiles ./dockerfiles
5556
5657ARG TIMESTAMP_SETTINGS=0
5758ARG TIMESTAMP_UWSGI=0
5859ARG VERSION_GITCOMMIT=unknown
5960
60−RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
61+RUN /usr/bin/python3 -m compileall -q searx; \
6162 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \
6263 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \
6364 if [ ! -z $VERSION_GITCOMMIT ]; then\
searx/autocomplete.py+6−6
@@ -24,6 +24,7 @@ from httpx import HTTPError
2424
2525
2626from searx import settings
27+from searx.data import ENGINES_LANGUAGES
2728from searx.network import get as http_get
2829from searx.exceptions import SearxEngineResponseException
2930
@@ -93,12 +94,11 @@ def google(query, lang):
9394
9495def startpage(query, lang):
9596 # startpage autocompleter
96− url = 'https://startpage.com/do/suggest?{query}'
97−
98− resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')
99− if len(resp) > 1:
100− return resp
101− return []
97+ lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')
98+ url = 'https://startpage.com/suggestions?{query}'
99+ resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))
100+ data = resp.json()
101+ return [e['text'] for e in data.get('suggestions', []) if 'text' in e]
102102
103103
104104def swisscows(query, lang):
searx/engines/gigablast.py+24−13
@@ -5,10 +5,10 @@
55# pylint: disable=missing-function-docstring, invalid-name
66
77import re
8−from json import loads
8+from json import loads, JSONDecodeError
99from urllib.parse import urlencode
10−# from searx import logger
1110from searx.network import get
11+from searx.exceptions import SearxEngineResponseException
1212
1313# about
1414about = {
@@ -22,6 +22,9 @@ about = {
2222
2323# engine dependent config
2424categories = ['general']
25+collections = 'main'
26+search_type = ''
27+fast = 0
2528# gigablast's pagination is totally damaged, don't use it
2629paging = False
2730safesearch = True
@@ -34,6 +37,8 @@ base_url = 'https://gigablast.com'
3437extra_param = ''
3538extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'
3639
40+_wait_for_results_msg = 'Loading results takes too long. Please enable fast option in gigablast engine.'
41+
3742def parse_extra_param(text):
3843
3944 # example:
@@ -54,7 +59,6 @@ def parse_extra_param(text):
5459 if re_var is not None and re_var.search(line):
5560 extra_param += re_var.search(line).group(1)
5661 break
57− # logger.debug('gigablast extra_param="%s"', extra_param)
5862
5963def init(engine_settings=None): # pylint: disable=unused-argument
6064 parse_extra_param(get(base_url + extra_param_path).text)
@@ -65,14 +69,17 @@ def request(query, params): # pylint: disable=unused-argument
6569
6670 # see API http://www.gigablast.com/api.html#/search
6771 # Take into account, that the API has some quirks ..
68−
69− query_args = dict(
70− c = 'main'
71− , format = 'json'
72− , q = query
73− , dr = 1
74− , showgoodimages = 0
75− )
72+ query_args = {
73+ 'c': collections,
74+ 'format': 'json',
75+ 'q': query,
76+ 'dr': 1 ,
77+ 'showgoodimages': 0,
78+ 'fast': fast,
79+ }
80+
81+ if search_type != '':
82+ query_args['searchtype'] = search_type
7683
7784 if params['language'] and params['language'] != 'all':
7885 query_args['qlangcountry'] = params['language']
@@ -90,9 +97,13 @@ def request(query, params): # pylint: disable=unused-argument
9097def response(resp):
9198 results = []
9299
93− response_json = loads(resp.text)
100+ try:
101+ response_json = loads(resp.text)
102+ except JSONDecodeError as e:
103+ if 'Waiting for results' in resp.text:
104+ raise SearxEngineResponseException(message=_wait_for_results_msg) # pylint: disable=raise-missing-from
105+ raise e
94106
95− # logger.debug('gigablast returns %s results', len(response_json['results']))
96107
97108 for result in response_json['results']:
98109 # see "Example JSON Output (&format=json)"
searx/network/__init__.py+22−1
@@ -5,6 +5,7 @@ import threading
55import concurrent.futures
66from time import time
77from queue import SimpleQueue
8+from types import MethodType
89
910import httpx
1011import h2.exceptions
@@ -134,15 +135,27 @@ async def stream_chunk_to_queue(network, q, method, url, **kwargs):
134135 try:
135136 async with await network.stream(method, url, **kwargs) as response:
136137 q.put(response)
138+ # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding
139+ # https://www.python-httpx.org/quickstart/#streaming-responses
137140 async for chunk in response.aiter_bytes(65536):
138141 if len(chunk) > 0:
139142 q.put(chunk)
143+ except httpx.ResponseClosed as e:
144+ # the response was closed
145+ pass
140146 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:
141147 q.put(e)
142148 finally:
143149 q.put(None)
144150
145151
152+def _close_response_method(self):
153+ asyncio.run_coroutine_threadsafe(
154+ self.aclose(),
155+ get_loop()
156+ )
157+
158+
146159def stream(method, url, **kwargs):
147160 """Replace httpx.stream.
148161
@@ -158,10 +171,18 @@ def stream(method, url, **kwargs):
158171 q = SimpleQueue()
159172 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),
160173 get_loop())
174+ # yield response
175+ response = q.get()
176+ if isinstance(response, Exception):
177+ raise response
178+ response.close = MethodType(_close_response_method, response)
179+ yield response
180+
181+ # yield chunks
161182 chunk_or_exception = q.get()
162183 while chunk_or_exception is not None:
163184 if isinstance(chunk_or_exception, Exception):
164185 raise chunk_or_exception
165186 yield chunk_or_exception
166187 chunk_or_exception = q.get()
167− return future.result()
188+ future.result()
searx/network/network.py+8
@@ -326,6 +326,14 @@ def initialize(settings_engines=None, settings_outgoing=None):
326326 if isinstance(network, str):
327327 NETWORKS[engine_name] = NETWORKS[network]
328328
329+ # the /image_proxy endpoint has a dedicated network.
330+ # same parameters than the default network, but HTTP/2 is disabled.
331+ # It decreases the CPU load average, and the total time is more or less the same
332+ if 'image_proxy' not in NETWORKS:
333+ image_proxy_params = default_params.copy()
334+ image_proxy_params['enable_http2'] = False
335+ NETWORKS['image_proxy'] = new_network(image_proxy_params)
336+
329337
330338@atexit.register
331339def done():
searx/settings.yml+12−8
@@ -505,6 +505,13 @@ engines:
505505 engine : gigablast
506506 shortcut : gb
507507 timeout : 3.0
508+ # Collection names to search in, separated by whitespaces.
509+ # By default searx, queries the main collection.
510+ #collections: main
511+ # Search types, not required. possible values: images, news
512+ #search_type: images
513+ # Enable fast to get results quicker.
514+ #fast : 1
508515 disabled: True
509516 additional_tests:
510517 rosebud: *test_rosebud
@@ -690,22 +697,19 @@ engines:
690697
691698 - name : hoogle
692699 engine : xpath
693− paging : True
694700 search_url : https://hoogle.haskell.org/?hoogle={query}&start={pageno}
695701 results_xpath : '//div[@class="result"]'
696− title_xpath : './/div[@class="ans"]//a'
697− url_xpath : './/div[@class="ans"]//a/@href'
698− content_xpath : './/div[@class="from"]'
699− page_size : 20
702+ title_xpath : './div[@class="ans"]'
703+ url_xpath : './div[@class="ans"]//a/@href'
704+ content_xpath : './div[contains(@class, "doc")]'
700705 categories : it
701706 shortcut : ho
702707 about:
703− website: https://www.haskell.org/
708+ website: https://hoogle.haskell.org/
704709 wikidata_id: Q34010
705− official_api_documentation: https://hackage.haskell.org/api
706710 use_official_api: false
707711 require_api_key: false
708− results: JSON
712+ results: HTML
709713
710714 - name : imdb
711715 engine : imdb
searx/utils.py+1−5
@@ -272,11 +272,7 @@ def dict_subset(d, properties):
272272 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])
273273 {'A': 'a'}
274274 """
275− result = {}
276− for k in properties:
277− if k in d:
278− result[k] = d[k]
279− return result
275+ return {k: d[k] for k in properties if k in d}
280276
281277
282278def get_torrent_size(filesize, filesize_multiplier):
searx/webapp.py+41−13
@@ -94,7 +94,7 @@ from searx.plugins import plugins
9494from searx.plugins.oa_doi_rewrite import get_doi_resolver
9595from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
9696from searx.answerers import answerers
97−from searx.network import stream as http_stream
97+from searx.network import stream as http_stream, set_context_network_name
9898from searx.answerers import ask
9999from searx.metrology.error_recorder import errors_per_engines
100100from searx.settings_loader import get_default_settings_path
@@ -921,6 +921,8 @@ def _is_selected_language_supported(engine, preferences):
921921
922922@app.route('/image_proxy', methods=['GET'])
923923def image_proxy():
924+ # pylint: disable=too-many-return-statements, too-many-branches
925+
924926 url = request.args.get('url')
925927
926928 if not url:
@@ -932,14 +934,21 @@ def image_proxy():
932934 return '', 400
933935
934936 maximum_size = 5 * 1024 * 1024
935−
937+ forward_resp = False
938+ resp = None
936939 try:
937− headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})
938− headers['User-Agent'] = gen_useragent()
940+ request_headers = {
941+ 'User-Agent': gen_useragent(),
942+ 'Accept': 'image/webp,*/*',
943+ 'Accept-Encoding': 'gzip, deflate',
944+ 'Sec-GPC': '1',
945+ 'DNT': '1',
946+ }
947+ set_context_network_name('image_proxy')
939948 stream = http_stream(
940949 method='GET',
941950 url=url,
942− headers=headers,
951+ headers=request_headers,
943952 timeout=settings['outgoing']['request_timeout'],
944953 follow_redirects=True,
945954 max_redirects=20)
@@ -949,25 +958,37 @@ def image_proxy():
949958 if content_length and content_length.isdigit() and int(content_length) > maximum_size:
950959 return 'Max size', 400
951960
952− if resp.status_code == 304:
953− return '', resp.status_code
954−
955961 if resp.status_code != 200:
956962 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))
957963 if resp.status_code >= 400:
958964 return '', resp.status_code
959965 return '', 400
960966
961− if not resp.headers.get('content-type', '').startswith('image/'):
962− logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
967+ if not resp.headers.get('Content-Type', '').startswith('image/'):
968+ logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', ''))
963969 return '', 400
964970
965− headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})
971+ forward_resp = True
972+ except httpx.HTTPError:
973+ logger.exception('HTTP error')
974+ return '', 400
975+ finally:
976+ if resp and not forward_resp:
977+ # the code is about to return an HTTP 400 error to the browser
978+ # we make sure to close the response between searxng and the HTTP server
979+ try:
980+ resp.close()
981+ except httpx.HTTPError:
982+ logger.exception('HTTP error on closing')
966983
967− total_length = 0
984+ try:
985+ headers = dict_subset(
986+ resp.headers,
987+ {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'}
988+ )
968989
969990 def forward_chunk():
970− nonlocal total_length
991+ total_length = 0
971992 for chunk in stream:
972993 total_length += len(chunk)
973994 if total_length > maximum_size:
@@ -1148,6 +1169,13 @@ def run():
11481169 )
11491170
11501171
1172+def patch_application(app):
1173+ # serve pages with HTTP/1.1
1174+ WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])
1175+ # patch app to handle non root url-s behind proxy & wsgi
1176+ app.wsgi_app = ReverseProxyPathFix(ProxyFix(app.wsgi_app))
1177+
1178+
11511179class ReverseProxyPathFix:
11521180 '''Wrap the application in this middleware and configure the
11531181 front-end server to add these headers, to let you quietly bind