ea38fea · searx · git.iko.soy

git.iko.soy/searx

Pick image_proxy changes from searxng (#2965)

* [mod] /image_proxy: don't decompress images * [fix] image_proxy: always close the httpx respone previously, when the content type was not an image and some other error, the httpx response was not closed * [mod] /image_proxy: use HTTP/1 instead of HTTP/2 httpx: HTTP/2 is slow when a lot data is downloaded. https://github.com/dalf/pyhttp-benchmark also, the usage of HTTP/1 decreases the load average * [mod] searx.utils.dict_subset: rewrite with comprehension Co-authored-by: Alexandre Flament <alex@al-f.net>

commitea38fea711f10dc3572f0fa20c26ddd3584870c1 authorNoémi Ványi <kvch@users.noreply.github.com> dateSat 22 Jan 2022, 13:49 (5 years ago) parentad7e00a

4 files changed+72−19

compare with master

unifiedsplit

searx/network/__init__.py+22−1 searx/network/network.py+8 searx/utils.py+1−5 searx/webapp.py+41−13

searx/network/__init__.py+22−1

@@ -5,6 +5,7 @@ import threading

55import concurrent.futures

66from time import time

77from queue import SimpleQueue

8+from types import MethodType

910import httpx

1011import h2.exceptions

@@ -134,15 +135,27 @@ async def stream_chunk_to_queue(network, q, method, url, **kwargs):

134135 try:

135136 async with await network.stream(method, url, **kwargs) as response:

136137 q.put(response)

138+ # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding

139+ # https://www.python-httpx.org/quickstart/#streaming-responses

137140 async for chunk in response.aiter_bytes(65536):

138141 if len(chunk) > 0:

139142 q.put(chunk)

143+ except httpx.ResponseClosed as e:

144+ # the response was closed

145+ pass

140146 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

141147 q.put(e)

142148 finally:

143149 q.put(None)

144150

145151

152+def _close_response_method(self):

153+ asyncio.run_coroutine_threadsafe(

154+ self.aclose(),

155+ get_loop()

156+ )

157+

158+

146159def stream(method, url, **kwargs):

147160 """Replace httpx.stream.

148161

@@ -158,10 +171,18 @@ def stream(method, url, **kwargs):

158171 q = SimpleQueue()

159172 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

160173 get_loop())

174+ # yield response

175+ response = q.get()

176+ if isinstance(response, Exception):

177+ raise response

178+ response.close = MethodType(_close_response_method, response)

179+ yield response

180+

181+ # yield chunks

161182 chunk_or_exception = q.get()

162183 while chunk_or_exception is not None:

163184 if isinstance(chunk_or_exception, Exception):

164185 raise chunk_or_exception

165186 yield chunk_or_exception

166187 chunk_or_exception = q.get()

167− return future.result()

188+ future.result()

searx/network/network.py+8

@@ -326,6 +326,14 @@ def initialize(settings_engines=None, settings_outgoing=None):

326326 if isinstance(network, str):

327327 NETWORKS[engine_name] = NETWORKS[network]

328328

329+ # the /image_proxy endpoint has a dedicated network.

330+ # same parameters than the default network, but HTTP/2 is disabled.

331+ # It decreases the CPU load average, and the total time is more or less the same

332+ if 'image_proxy' not in NETWORKS:

333+ image_proxy_params = default_params.copy()

334+ image_proxy_params['enable_http2'] = False

335+ NETWORKS['image_proxy'] = new_network(image_proxy_params)

336+

329337

330338@atexit.register

331339def done():

searx/utils.py+1−5

@@ -272,11 +272,7 @@ def dict_subset(d, properties):

272272 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

273273 {'A': 'a'}

274274 """

275− result = {}

276− for k in properties:

277− if k in d:

278− result[k] = d[k]

279− return result

275+ return {k: d[k] for k in properties if k in d}

280276

281277

282278def get_torrent_size(filesize, filesize_multiplier):

searx/webapp.py+41−13

@@ -94,7 +94,7 @@ from searx.plugins import plugins

9494from searx.plugins.oa_doi_rewrite import get_doi_resolver

9595from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

9696from searx.answerers import answerers

97−from searx.network import stream as http_stream

97+from searx.network import stream as http_stream, set_context_network_name

9898from searx.answerers import ask

9999from searx.metrology.error_recorder import errors_per_engines

100100from searx.settings_loader import get_default_settings_path

@@ -921,6 +921,8 @@ def _is_selected_language_supported(engine, preferences):

921921

922922@app.route('/image_proxy', methods=['GET'])

923923def image_proxy():

924+ # pylint: disable=too-many-return-statements, too-many-branches

925+

924926 url = request.args.get('url')

925927

926928 if not url:

@@ -932,14 +934,21 @@ def image_proxy():

932934 return '', 400

933935

934936 maximum_size = 5 * 1024 * 1024

935−

937+ forward_resp = False

938+ resp = None

936939 try:

937− headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})

938− headers['User-Agent'] = gen_useragent()

940+ request_headers = {

941+ 'User-Agent': gen_useragent(),

942+ 'Accept': 'image/webp,*/*',

943+ 'Accept-Encoding': 'gzip, deflate',

944+ 'Sec-GPC': '1',

945+ 'DNT': '1',

946+ }

947+ set_context_network_name('image_proxy')

939948 stream = http_stream(

940949 method='GET',

941950 url=url,

942− headers=headers,

951+ headers=request_headers,

943952 timeout=settings['outgoing']['request_timeout'],

944953 follow_redirects=True,

945954 max_redirects=20)

@@ -949,25 +958,37 @@ def image_proxy():

949958 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

950959 return 'Max size', 400

951960

952− if resp.status_code == 304:

953− return '', resp.status_code

954−

955961 if resp.status_code != 200:

956962 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

957963 if resp.status_code >= 400:

958964 return '', resp.status_code

959965 return '', 400

960966

961− if not resp.headers.get('content-type', '').startswith('image/'):

962− logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))

967+ if not resp.headers.get('Content-Type', '').startswith('image/'):

968+ logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', ''))

963969 return '', 400

964970

965− headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

971+ forward_resp = True

972+ except httpx.HTTPError:

973+ logger.exception('HTTP error')

974+ return '', 400

975+ finally:

976+ if resp and not forward_resp:

977+ # the code is about to return an HTTP 400 error to the browser

978+ # we make sure to close the response between searxng and the HTTP server

979+ try:

980+ resp.close()

981+ except httpx.HTTPError:

982+ logger.exception('HTTP error on closing')

966983

967− total_length = 0

984+ try:

985+ headers = dict_subset(

986+ resp.headers,

987+ {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'}

988+ )

968989

969990 def forward_chunk():

970− nonlocal total_length

991+ total_length = 0

971992 for chunk in stream:

972993 total_length += len(chunk)

973994 if total_length > maximum_size:

@@ -1148,6 +1169,13 @@ def run():

11481169 )

11491170

11501171

1172+def patch_application(app):

1173+ # serve pages with HTTP/1.1

1174+ WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])

1175+ # patch app to handle non root url-s behind proxy & wsgi

1176+ app.wsgi_app = ReverseProxyPathFix(ProxyFix(app.wsgi_app))

1177+

1178+

11511179class ReverseProxyPathFix:

11521180 '''Wrap the application in this middleware and configure the

11531181 front-end server to add these headers, to let you quietly bind