ea38fea · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

Pick image_proxy changes from searxng (#2965)

ea38feaNoémi Ványi5 years ago

searx/network/__init__.py+22−1

compare with master

unifiedsplit

@@ -5,6 +5,7 @@ import threading

5import concurrent.futures

6from time import time

7from queue import SimpleQueue

8+from types import MethodType

9import httpx

10import httpx

10import h2.exceptions

11import h2.exceptions

@@ -134,15 +135,27 @@ async def stream_chunk_to_queue(network, q, method, url, **kwargs):

134 try:

135 try:

135 async with await network.stream(method, url, **kwargs) as response:

136 async with await network.stream(method, url, **kwargs) as response:

136 q.put(response)

137 q.put(response)

138+ # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding

139+ # https://www.python-httpx.org/quickstart/#streaming-responses

137 async for chunk in response.aiter_bytes(65536):

140 async for chunk in response.aiter_bytes(65536):

138 if len(chunk) > 0:

141 if len(chunk) > 0:

139 q.put(chunk)

142 q.put(chunk)

143+ except httpx.ResponseClosed as e:

144+ # the response was closed

145+ pass

140 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

146 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

141 q.put(e)

147 q.put(e)

142 finally:

148 finally:

143 q.put(None)

149 q.put(None)

144

150

145

151

152+def _close_response_method(self):

153+ asyncio.run_coroutine_threadsafe(

154+ self.aclose(),

155+ get_loop()

156+ )

157+

158+

146def stream(method, url, **kwargs):

159def stream(method, url, **kwargs):

147 """Replace httpx.stream.

160 """Replace httpx.stream.

148

161

@@ -158,10 +171,18 @@ def stream(method, url, **kwargs):

158 q = SimpleQueue()

171 q = SimpleQueue()

159 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

172 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

160 get_loop())

173 get_loop())

174+ # yield response

175+ response = q.get()

176+ if isinstance(response, Exception):

177+ raise response

178+ response.close = MethodType(_close_response_method, response)

179+ yield response

180+

181+ # yield chunks

161 chunk_or_exception = q.get()

182 chunk_or_exception = q.get()

162 while chunk_or_exception is not None:

183 while chunk_or_exception is not None:

163 if isinstance(chunk_or_exception, Exception):

184 if isinstance(chunk_or_exception, Exception):

164 raise chunk_or_exception

185 raise chunk_or_exception

165 yield chunk_or_exception

186 yield chunk_or_exception

166 chunk_or_exception = q.get()

187 chunk_or_exception = q.get()

167− return future.result()

188+ future.result()

searx/network/network.py+8

@@ -326,6 +326,14 @@ def initialize(settings_engines=None, settings_outgoing=None):

326 if isinstance(network, str):

327 NETWORKS[engine_name] = NETWORKS[network]

328

329+ # the /image_proxy endpoint has a dedicated network.

330+ # same parameters than the default network, but HTTP/2 is disabled.

331+ # It decreases the CPU load average, and the total time is more or less the same

332+ if 'image_proxy' not in NETWORKS:

333+ image_proxy_params = default_params.copy()

334+ image_proxy_params['enable_http2'] = False

335+ NETWORKS['image_proxy'] = new_network(image_proxy_params)

336+

329

337

330@atexit.register

338@atexit.register

331def done():

339def done():

searx/utils.py+1−5

@@ -272,11 +272,7 @@ def dict_subset(d, properties):

272 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

273 {'A': 'a'}

274 """

275− result = {}

276− for k in properties:

275+ return {k: d[k] for k in properties if k in d}

277− if k in d:

278− result[k] = d[k]

279− return result

280

276

281

277

282def get_torrent_size(filesize, filesize_multiplier):

278def get_torrent_size(filesize, filesize_multiplier):

searx/webapp.py+41−13

@@ -94,7 +94,7 @@ from searx.plugins import plugins

94from searx.plugins.oa_doi_rewrite import get_doi_resolver

95from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

96from searx.answerers import answerers

97−from searx.network import stream as http_stream

97+from searx.network import stream as http_stream, set_context_network_name

98from searx.answerers import ask

99from searx.metrology.error_recorder import errors_per_engines

100from searx.settings_loader import get_default_settings_path

@@ -921,6 +921,8 @@ def _is_selected_language_supported(engine, preferences):

921

922@app.route('/image_proxy', methods=['GET'])

923def image_proxy():

924+ # pylint: disable=too-many-return-statements, too-many-branches

925+

924 url = request.args.get('url')

926 url = request.args.get('url')

925

927

926 if not url:

928 if not url:

@@ -932,14 +934,21 @@ def image_proxy():

932 return '', 400

934 return '', 400

933

935

934 maximum_size = 5 * 1024 * 1024

936 maximum_size = 5 * 1024 * 1024

935−

937+ forward_resp = False

938+ resp = None

936 try:

939 try:

937− headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})

940+ request_headers = {

938− headers['User-Agent'] = gen_useragent()

941+ 'User-Agent': gen_useragent(),

942+ 'Accept': 'image/webp,*/*',

943+ 'Accept-Encoding': 'gzip, deflate',

944+ 'Sec-GPC': '1',

945+ 'DNT': '1',

946+ }

947+ set_context_network_name('image_proxy')

939 stream = http_stream(

948 stream = http_stream(

940 method='GET',

949 method='GET',

941 url=url,

950 url=url,

942− headers=headers,

951+ headers=request_headers,

943 timeout=settings['outgoing']['request_timeout'],

952 timeout=settings['outgoing']['request_timeout'],

944 follow_redirects=True,

953 follow_redirects=True,

945 max_redirects=20)

954 max_redirects=20)

@@ -949,25 +958,37 @@ def image_proxy():

949 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

958 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

950 return 'Max size', 400

959 return 'Max size', 400

951

960

952− if resp.status_code == 304:

953− return '', resp.status_code

954−

955 if resp.status_code != 200:

961 if resp.status_code != 200:

956 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

962 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

957 if resp.status_code >= 400:

963 if resp.status_code >= 400:

958 return '', resp.status_code

964 return '', resp.status_code

959 return '', 400

965 return '', 400

960

966

961− if not resp.headers.get('content-type', '').startswith('image/'):

967+ if not resp.headers.get('Content-Type', '').startswith('image/'):

962− logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))

968+ logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', ''))

963 return '', 400

969 return '', 400

964

970

965− headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

971+ forward_resp = True

972+ except httpx.HTTPError:

973+ logger.exception('HTTP error')

974+ return '', 400

975+ finally:

976+ if resp and not forward_resp:

977+ # the code is about to return an HTTP 400 error to the browser

978+ # we make sure to close the response between searxng and the HTTP server

979+ try:

980+ resp.close()

981+ except httpx.HTTPError:

982+ logger.exception('HTTP error on closing')

966

983

967− total_length = 0

984+ try:

985+ headers = dict_subset(

986+ resp.headers,

987+ {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'}

988+ )

968

989

969 def forward_chunk():

990 def forward_chunk():

970− nonlocal total_length

991+ total_length = 0

971 for chunk in stream:

992 for chunk in stream:

972 total_length += len(chunk)

993 total_length += len(chunk)

973 if total_length > maximum_size:

994 if total_length > maximum_size:

@@ -1148,6 +1169,13 @@ def run():

1148 )

1169 )

1149

1170

1150

1171

1172+def patch_application(app):

1173+ # serve pages with HTTP/1.1

1174+ WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])

1175+ # patch app to handle non root url-s behind proxy & wsgi

1176+ app.wsgi_app = ReverseProxyPathFix(ProxyFix(app.wsgi_app))

1177+

1178+

1151class ReverseProxyPathFix:

1179class ReverseProxyPathFix:

1152 '''Wrap the application in this middleware and configure the

1180 '''Wrap the application in this middleware and configure the

1153 front-end server to add these headers, to let you quietly bind

1181 front-end server to add these headers, to let you quietly bind