master..a6184ac · searx

Dockerfile+2−3

since splitsnapshots

unifiedsplit

@@ -51,14 +51,13 @@ RUN apk upgrade --no-cache \

51 && apk del build-dependencies \

52 && rm -rf /root/.cache

53

54−COPY searx ./searx

54+COPY --chown=searx:searx . .

55−COPY dockerfiles ./dockerfiles

56

55

57ARG TIMESTAMP_SETTINGS=0

56ARG TIMESTAMP_SETTINGS=0

58ARG TIMESTAMP_UWSGI=0

57ARG TIMESTAMP_UWSGI=0

59ARG VERSION_GITCOMMIT=unknown

58ARG VERSION_GITCOMMIT=unknown

60

59

61−RUN /usr/bin/python3 -m compileall -q searx; \

60+RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

62 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

61 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

63 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

62 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

64 if [ ! -z $VERSION_GITCOMMIT ]; then\

63 if [ ! -z $VERSION_GITCOMMIT ]; then\

searx/autocomplete.py+6−6

@@ -24,7 +24,6 @@ from httpx import HTTPError

24

25

26from searx import settings

27−from searx.data import ENGINES_LANGUAGES

28from searx.network import get as http_get

27from searx.network import get as http_get

29from searx.exceptions import SearxEngineResponseException

28from searx.exceptions import SearxEngineResponseException

30

29

@@ -94,11 +93,12 @@ def google(query, lang):

94

93

95def startpage(query, lang):

94def startpage(query, lang):

96 # startpage autocompleter

95 # startpage autocompleter

97− lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')

98− url = 'https://startpage.com/suggestions?{query}'

96+ url = 'https://startpage.com/do/suggest?{query}'

97+

99− resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))

98+ resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')

100− data = resp.json()

99+ if len(resp) > 1:

101− return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

100+ return resp

101+ return []

102

103

104def swisscows(query, lang):

searx/engines/gigablast.py+13−24

@@ -5,10 +5,10 @@

5# pylint: disable=missing-function-docstring, invalid-name

6

7import re

8−from json import loads, JSONDecodeError

8+from json import loads

9from urllib.parse import urlencode

10+# from searx import logger

10from searx.network import get

11from searx.network import get

11−from searx.exceptions import SearxEngineResponseException

12

13# about

14about = {

@@ -22,9 +22,6 @@ about = {

22

23# engine dependent config

24categories = ['general']

25−collections = 'main'

26−search_type = ''

27−fast = 0

28# gigablast's pagination is totally damaged, don't use it

25# gigablast's pagination is totally damaged, don't use it

29paging = False

26paging = False

30safesearch = True

27safesearch = True

@@ -37,8 +34,6 @@ base_url = 'https://gigablast.com'

37extra_param = ''

34extra_param = ''

38extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'

35extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'

39

36

40−_wait_for_results_msg = 'Loading results takes too long. Please enable fast option in gigablast engine.'

41−

42def parse_extra_param(text):

37def parse_extra_param(text):

43

38

44 # example:

39 # example:

@@ -59,6 +54,7 @@ def parse_extra_param(text):

59 if re_var is not None and re_var.search(line):

54 if re_var is not None and re_var.search(line):

60 extra_param += re_var.search(line).group(1)

55 extra_param += re_var.search(line).group(1)

61 break

56 break

57+ # logger.debug('gigablast extra_param="%s"', extra_param)

62

58

63def init(engine_settings=None): # pylint: disable=unused-argument

59def init(engine_settings=None): # pylint: disable=unused-argument

64 parse_extra_param(get(base_url + extra_param_path).text)

60 parse_extra_param(get(base_url + extra_param_path).text)

@@ -69,17 +65,14 @@ def request(query, params): # pylint: disable=unused-argument

69

65

70 # see API http://www.gigablast.com/api.html#/search

66 # see API http://www.gigablast.com/api.html#/search

71 # Take into account, that the API has some quirks ..

67 # Take into account, that the API has some quirks ..

68+

72− query_args = {

69+ query_args = dict(

73− 'c': collections,

70+ c = 'main'

74− 'format': 'json',

71+ , format = 'json'

75− 'q': query,

72+ , q = query

76− 'dr': 1 ,

73+ , dr = 1

77− 'showgoodimages': 0,

74+ , showgoodimages = 0

78− 'fast': fast,

75+ )

79− }

80−

81− if search_type != '':

82− query_args['searchtype'] = search_type

83

76

84 if params['language'] and params['language'] != 'all':

77 if params['language'] and params['language'] != 'all':

85 query_args['qlangcountry'] = params['language']

78 query_args['qlangcountry'] = params['language']

@@ -97,13 +90,9 @@ def request(query, params): # pylint: disable=unused-argument

97def response(resp):

90def response(resp):

98 results = []

91 results = []

99

92

100− try:

101− response_json = loads(resp.text)

93+ response_json = loads(resp.text)

102− except JSONDecodeError as e:

103− if 'Waiting for results' in resp.text:

104− raise SearxEngineResponseException(message=_wait_for_results_msg) # pylint: disable=raise-missing-from

105− raise e

106

94

95+ # logger.debug('gigablast returns %s results', len(response_json['results']))

107

96

108 for result in response_json['results']:

97 for result in response_json['results']:

109 # see "Example JSON Output (&format=json)"

98 # see "Example JSON Output (&format=json)"

searx/engines/google.py+1−1

@@ -8,7 +8,7 @@ Definitions`_.

8 https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions

9"""

10

11−# pylint: disable=invalid-name, missing-function-docstring, too-many-branches

11+# pylint: disable=invalid-name, missing-function-docstring

12

13from urllib.parse import urlencode

14from lxml import html

searx/engines/startpage.py+1−68

@@ -5,7 +5,6 @@

5"""

6

7import re

8−from time import time

9

8

10from urllib.parse import urlencode

9from urllib.parse import urlencode

11from unicodedata import normalize, combining

10from unicodedata import normalize, combining

@@ -16,15 +15,7 @@ from lxml import html

16from babel import Locale

15from babel import Locale

17from babel.localedata import locale_identifiers

16from babel.localedata import locale_identifiers

18

17

19−from searx import logger

20−from searx.network import get

21from searx.utils import extract_text, eval_xpath, match_language

18from searx.utils import extract_text, eval_xpath, match_language

22−from searx.exceptions import (

23− SearxEngineResponseException,

24− SearxEngineCaptchaException,

25−)

26−

27−logger = logger.getChild('startpage')

28

19

29# about

20# about

30about = {

21about = {

@@ -56,74 +47,16 @@ results_xpath = '//div[@class="w-gl__result__main"]'

56link_xpath = './/a[@class="w-gl__result-title result-link"]'

47link_xpath = './/a[@class="w-gl__result-title result-link"]'

57content_xpath = './/p[@class="w-gl__description"]'

48content_xpath = './/p[@class="w-gl__description"]'

58

49

59−# timestamp of the last fetch of 'sc' code

60−sc_code_ts = 0

61−sc_code = ''

62−

63−

64−def raise_captcha(resp):

65−

66− if str(resp.url).startswith('https://www.startpage.com/sp/captcha'):

67− # suspend CAPTCHA for 7 days

68− raise SearxEngineCaptchaException(suspended_time=7 * 24 * 3600)

69−

70−

71−def get_sc_code(headers):

72− """Get an actual `sc` argument from startpage's home page.

73−

74− Startpage puts a `sc` argument on every link. Without this argument

75− startpage considers the request is from a bot. We do not know what is

76− encoded in the value of the `sc` argument, but it seems to be a kind of a

77− *time-stamp*. This *time-stamp* is valid for a few hours.

78−

79− This function scrap a new *time-stamp* from startpage's home page every hour

80− (3000 sec).

81−

82− """

83−

84− global sc_code_ts, sc_code # pylint: disable=global-statement

85−

86− if time() > (sc_code_ts + 3000):

87− logger.debug("query new sc time-stamp ...")

88−

89− resp = get(base_url, headers=headers)

90− raise_captcha(resp)

91− dom = html.fromstring(resp.text)

92−

93− try:

94− # href --> '/?sc=adrKJMgF8xwp20'

95− href = eval_xpath(dom, '//a[@class="footer-home__logo"]')[0].get('href')

96− except IndexError as exc:

97− # suspend startpage API --> https://github.com/searxng/searxng/pull/695

98− raise SearxEngineResponseException(

99− suspended_time=7 * 24 * 3600, message="PR-695: query new sc time-stamp failed!"

100− ) from exc

101−

102− sc_code = href[5:]

103− sc_code_ts = time()

104− logger.debug("new value is: %s", sc_code)

105−

106− return sc_code

107−

108

50

109# do search-request

51# do search-request

110def request(query, params):

52def request(query, params):

111

53

112− # pylint: disable=line-too-long

113− # The format string from Startpage's FFox add-on [1]::

114− #

115− # https://www.startpage.com/do/dsearch?query={searchTerms}&cat=web&pl=ext-ff&language=__MSG_extensionUrlLanguage__&extVersion=1.3.0

116− #

117− # [1] https://addons.mozilla.org/en-US/firefox/addon/startpage-private-search/

118−

119 args = {

54 args = {

120 'query': query,

55 'query': query,

121 'page': params['pageno'],

56 'page': params['pageno'],

122 'cat': 'web',

57 'cat': 'web',

123− # 'pl': 'ext-ff',

124− # 'extVersion': '1.3.0',

125 # 'abp': "-1",

58 # 'abp': "-1",

126− 'sc': get_sc_code(params['headers']),

59+ 'sc': 'Mj4jZy61QETj20',

127 }

60 }

128

61

129 # set language if specified

62 # set language if specified

searx/engines/tineye.pydeleted−86

@@ -1,86 +0,0 @@

1−"""

2−Tineye - Reverse search images

3−"""

4−

5−from json import loads

6−from urllib.parse import urlencode

7−

8−from datetime import datetime

9−

10−about = {

11− "website": "https://tineye.com",

12− "wikidata_id": "Q2382535",

13− "use_official_api": False,

14− "require_api_key": False,

15− "results": "JSON",

16−}

17−

18−

19−categories = ['images']

20−paging = True

21−

22−safesearch = False

23−

24−

25−base_url = 'https://tineye.com'

26−search_string = '/result_json/?page={page}&{query}'

27−

28−

29−def request(query, params):

30− params['url'] = base_url +\

31− search_string.format(

32− query=urlencode({'url': query}),

33− page=params['pageno'])

34−

35− params['headers'].update({

36− 'Connection': 'keep-alive',

37− 'Accept-Encoding': 'gzip, defalte, br',

38− 'Host': 'tineye.com',

39− 'DNT': '1',

40− 'TE': 'trailers',

41− })

42−

43− return params

44−

45−

46−def response(resp):

47− results = []

48− # Define wanted results

49− json_data = loads(resp.text)

50− number_of_results = json_data['num_matches']

51−

52− for i in json_data['matches']:

53− for i in json_data['matches']:

54− image_format = i['format']

55− width = i['width']

56− height = i['height']

57− thumbnail_src = i['image_url']

58− backlink = i['domains'][0]['backlinks'][0]

59−

60− url = backlink['backlink']

61− source = backlink['url']

62− title = backlink['image_name']

63− img_src = backlink['url']

64−

65− # Get and convert published date

66− api_date = backlink['crawl_date'][:-3]

67− publishedDate = datetime.fromisoformat(api_date)

68−

69− # Append results

70− results.append({

71− 'template': 'images.html',

72− 'url': url,

73− 'thumbnail_src': thumbnail_src,

74− 'source': source,

75− 'title': title,

76− 'img_src': img_src,

77− 'format': image_format,

78− 'widht': width,

79− 'height': height,

80− 'publishedDate': publishedDate,

81− })

82−

83− # Append number of results

84− results.append({'number_of_results': number_of_results})

85−

86− return results

searx/network/__init__.py+1−22

@@ -5,7 +5,6 @@ import threading

5import concurrent.futures

6from time import time

7from queue import SimpleQueue

8−from types import MethodType

9

8

10import httpx

9import httpx

11import h2.exceptions

10import h2.exceptions

@@ -135,27 +134,15 @@ async def stream_chunk_to_queue(network, q, method, url, **kwargs):

135 try:

134 try:

136 async with await network.stream(method, url, **kwargs) as response:

135 async with await network.stream(method, url, **kwargs) as response:

137 q.put(response)

136 q.put(response)

138− # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding

139− # https://www.python-httpx.org/quickstart/#streaming-responses

140 async for chunk in response.aiter_bytes(65536):

137 async for chunk in response.aiter_bytes(65536):

141 if len(chunk) > 0:

138 if len(chunk) > 0:

142 q.put(chunk)

139 q.put(chunk)

143− except httpx.ResponseClosed as e:

144− # the response was closed

145− pass

146 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

140 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

147 q.put(e)

141 q.put(e)

148 finally:

142 finally:

149 q.put(None)

143 q.put(None)

150

144

151

145

152−def _close_response_method(self):

153− asyncio.run_coroutine_threadsafe(

154− self.aclose(),

155− get_loop()

156− )

157−

158−

159def stream(method, url, **kwargs):

146def stream(method, url, **kwargs):

160 """Replace httpx.stream.

147 """Replace httpx.stream.

161

148

@@ -171,18 +158,10 @@ def stream(method, url, **kwargs):

171 q = SimpleQueue()

158 q = SimpleQueue()

172 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

159 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

173 get_loop())

160 get_loop())

174− # yield response

175− response = q.get()

176− if isinstance(response, Exception):

177− raise response

178− response.close = MethodType(_close_response_method, response)

179− yield response

180−

181− # yield chunks

182 chunk_or_exception = q.get()

161 chunk_or_exception = q.get()

183 while chunk_or_exception is not None:

162 while chunk_or_exception is not None:

184 if isinstance(chunk_or_exception, Exception):

163 if isinstance(chunk_or_exception, Exception):

185 raise chunk_or_exception

164 raise chunk_or_exception

186 yield chunk_or_exception

165 yield chunk_or_exception

187 chunk_or_exception = q.get()

166 chunk_or_exception = q.get()

188− future.result()

167+ return future.result()

searx/network/network.py−8

@@ -326,14 +326,6 @@ def initialize(settings_engines=None, settings_outgoing=None):

326 if isinstance(network, str):

327 NETWORKS[engine_name] = NETWORKS[network]

328

329− # the /image_proxy endpoint has a dedicated network.

330− # same parameters than the default network, but HTTP/2 is disabled.

331− # It decreases the CPU load average, and the total time is more or less the same

332− if 'image_proxy' not in NETWORKS:

333− image_proxy_params = default_params.copy()

334− image_proxy_params['enable_http2'] = False

335− NETWORKS['image_proxy'] = new_network(image_proxy_params)

336−

337

329

338@atexit.register

330@atexit.register

339def done():

331def done():

searx/results.py+2−8

@@ -6,7 +6,6 @@ from urllib.parse import urlparse, unquote

6from searx import logger

7from searx.engines import engines

8from searx.metrology.error_recorder import record_error

9−from searx.utils import add_scheme_to_url

10from searx import settings

9from searx import settings

11

10

12

11

@@ -241,15 +240,10 @@ class ResultContainer:

241 result['parsed_url'] = urlparse(result['url'])

240 result['parsed_url'] = urlparse(result['url'])

242

241

243 # if the result has no scheme, use http as default

242 # if the result has no scheme, use http as default

244− if not result['parsed_url'].scheme or result['parsed_url'].scheme == '':

243+ if not result['parsed_url'].scheme:

245− result['parsed_url'] = result['parsed_url']._replace(scheme='http')

244+ result['parsed_url'] = result['parsed_url']._replace(scheme="http")

246 result['url'] = result['parsed_url'].geturl()

245 result['url'] = result['parsed_url'].geturl()

247

246

248− if 'thumbnail_src' in result:

249− result['thumbnail_src'] = add_scheme_to_url(result['thumbnail_src'])

250− if 'img_src' in result:

251− result['img_src'] = add_scheme_to_url(result['img_src'])

252−

253 result['engines'] = set([result['engine']])

247 result['engines'] = set([result['engine']])

254

248

255 # strip multiple spaces and cariage returns from content

249 # strip multiple spaces and cariage returns from content

searx/settings.yml+8−19

@@ -410,13 +410,6 @@ engines:

410 timeout: 3.0

411 disabled : True

412

413− - name : tineye

414− engine : tineye

415− shortcut : tin

416− paging : True

417− timeout : 9.0

418− disabled : True

419−

420 - name : etools

413 - name : etools

421 engine : etools

414 engine : etools

422 shortcut : eto

415 shortcut : eto

@@ -505,13 +498,6 @@ engines:

505 engine : gigablast

498 engine : gigablast

506 shortcut : gb

499 shortcut : gb

507 timeout : 3.0

500 timeout : 3.0

508− # Collection names to search in, separated by whitespaces.

509− # By default searx, queries the main collection.

510− #collections: main

511− # Search types, not required. possible values: images, news

512− #search_type: images

513− # Enable fast to get results quicker.

514− #fast : 1

515 disabled: True

501 disabled: True

516 additional_tests:

502 additional_tests:

517 rosebud: *test_rosebud

503 rosebud: *test_rosebud

@@ -697,19 +683,22 @@ engines:

697

683

698 - name : hoogle

684 - name : hoogle

699 engine : xpath

685 engine : xpath

686+ paging : True

700 search_url : https://hoogle.haskell.org/?hoogle={query}&start={pageno}

687 search_url : https://hoogle.haskell.org/?hoogle={query}&start={pageno}

701 results_xpath : '//div[@class="result"]'

688 results_xpath : '//div[@class="result"]'

702− title_xpath : './div[@class="ans"]'

689+ title_xpath : './/div[@class="ans"]//a'

703− url_xpath : './div[@class="ans"]//a/@href'

690+ url_xpath : './/div[@class="ans"]//a/@href'

704− content_xpath : './div[contains(@class, "doc")]'

691+ content_xpath : './/div[@class="from"]'

692+ page_size : 20

705 categories : it

693 categories : it

706 shortcut : ho

694 shortcut : ho

707 about:

695 about:

708− website: https://hoogle.haskell.org/

696+ website: https://www.haskell.org/

709 wikidata_id: Q34010

697 wikidata_id: Q34010

698+ official_api_documentation: https://hackage.haskell.org/api

710 use_official_api: false

699 use_official_api: false

711 require_api_key: false

700 require_api_key: false

712− results: HTML

701+ results: JSON

713

702

714 - name : imdb

703 - name : imdb

715 engine : imdb

704 engine : imdb

searx/utils.py+6−12

@@ -7,7 +7,7 @@ from numbers import Number

7from os.path import splitext, join

8from random import choice

9from html.parser import HTMLParser

10−from urllib.parse import urljoin, urlparse, urlunparse

10+from urllib.parse import urljoin, urlparse

11

12from lxml import html

13from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult

@@ -214,16 +214,6 @@ def normalize_url(url, base_url):

214 return url

215

216

217−def add_scheme_to_url(url, scheme="https"):

218− """Add schema to URL: if scheme is missing from the URL, then add it."""

219−

220− parsed = urlparse(url)

221− if parsed.scheme == '':

222− parsed_with_scheme = parsed._replace(scheme=scheme)

223− return urlunparse(parsed_with_scheme)

224− return url

225−

226−

227def extract_url(xpath_results, base_url):

217def extract_url(xpath_results, base_url):

228 """Extract and normalize URL from lxml Element

218 """Extract and normalize URL from lxml Element

229

219

@@ -272,7 +262,11 @@ def dict_subset(d, properties):

272 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

262 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

273 {'A': 'a'}

263 {'A': 'a'}

274 """

264 """

265+ result = {}

275− return {k: d[k] for k in properties if k in d}

266+ for k in properties:

267+ if k in d:

268+ result[k] = d[k]

269+ return result

276

270

277

271

278def get_torrent_size(filesize, filesize_multiplier):

272def get_torrent_size(filesize, filesize_multiplier):

searx/webapp.py+13−41

@@ -94,7 +94,7 @@ from searx.plugins import plugins

94from searx.plugins.oa_doi_rewrite import get_doi_resolver

95from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

96from searx.answerers import answerers

97−from searx.network import stream as http_stream, set_context_network_name

97+from searx.network import stream as http_stream

98from searx.answerers import ask

99from searx.metrology.error_recorder import errors_per_engines

100from searx.settings_loader import get_default_settings_path

@@ -921,8 +921,6 @@ def _is_selected_language_supported(engine, preferences):

921

922@app.route('/image_proxy', methods=['GET'])

923def image_proxy():

924− # pylint: disable=too-many-return-statements, too-many-branches

925−

926 url = request.args.get('url')

924 url = request.args.get('url')

927

925

928 if not url:

926 if not url:

@@ -934,21 +932,14 @@ def image_proxy():

934 return '', 400

932 return '', 400

935

933

936 maximum_size = 5 * 1024 * 1024

934 maximum_size = 5 * 1024 * 1024

937− forward_resp = False

935+

938− resp = None

939 try:

936 try:

940− request_headers = {

937+ headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})

941− 'User-Agent': gen_useragent(),

938+ headers['User-Agent'] = gen_useragent()

942− 'Accept': 'image/webp,*/*',

943− 'Accept-Encoding': 'gzip, deflate',

944− 'Sec-GPC': '1',

945− 'DNT': '1',

946− }

947− set_context_network_name('image_proxy')

948 stream = http_stream(

939 stream = http_stream(

949 method='GET',

940 method='GET',

950 url=url,

941 url=url,

951− headers=request_headers,

942+ headers=headers,

952 timeout=settings['outgoing']['request_timeout'],

943 timeout=settings['outgoing']['request_timeout'],

953 follow_redirects=True,

944 follow_redirects=True,

954 max_redirects=20)

945 max_redirects=20)

@@ -958,37 +949,25 @@ def image_proxy():

958 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

949 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

959 return 'Max size', 400

950 return 'Max size', 400

960

951

952+ if resp.status_code == 304:

953+ return '', resp.status_code

954+

961 if resp.status_code != 200:

955 if resp.status_code != 200:

962 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

956 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

963 if resp.status_code >= 400:

957 if resp.status_code >= 400:

964 return '', resp.status_code

958 return '', resp.status_code

965 return '', 400

959 return '', 400

966

960

967− if not resp.headers.get('Content-Type', '').startswith('image/'):

961+ if not resp.headers.get('content-type', '').startswith('image/'):

968− logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', ''))

962+ logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))

969 return '', 400

963 return '', 400

970

964

971− forward_resp = True

965+ headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

972− except httpx.HTTPError:

973− logger.exception('HTTP error')

974− return '', 400

975− finally:

976− if resp and not forward_resp:

977− # the code is about to return an HTTP 400 error to the browser

978− # we make sure to close the response between searxng and the HTTP server

979− try:

980− resp.close()

981− except httpx.HTTPError:

982− logger.exception('HTTP error on closing')

983

966

984− try:

967+ total_length = 0

985− headers = dict_subset(

986− resp.headers,

987− {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'}

988− )

989

968

990 def forward_chunk():

969 def forward_chunk():

991− total_length = 0

970+ nonlocal total_length

992 for chunk in stream:

971 for chunk in stream:

993 total_length += len(chunk)

972 total_length += len(chunk)

994 if total_length > maximum_size:

973 if total_length > maximum_size:

@@ -1169,13 +1148,6 @@ def run():

1169 )

1148 )

1170

1149

1171

1150

1172−def patch_application(app):

1173− # serve pages with HTTP/1.1

1174− WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])

1175− # patch app to handle non root url-s behind proxy & wsgi

1176− app.wsgi_app = ReverseProxyPathFix(ProxyFix(app.wsgi_app))

1177−

1178−

1179class ReverseProxyPathFix:

1151class ReverseProxyPathFix:

1180 '''Wrap the application in this middleware and configure the

1152 '''Wrap the application in this middleware and configure the

1181 front-end server to add these headers, to let you quietly bind

1153 front-end server to add these headers, to let you quietly bind