0c351ea..master · searx

git.iko.soy/searx

on master help sign in

Dockerfile+3−2

since splitsnapshots

unifiedsplit

@@ -51,13 +51,14 @@ RUN apk upgrade --no-cache \

51 && apk del build-dependencies \

52 && rm -rf /root/.cache

54−COPY --chown=searx:searx . .

54+COPY searx ./searx

55+COPY dockerfiles ./dockerfiles

56ARG TIMESTAMP_SETTINGS=0

57ARG TIMESTAMP_SETTINGS=0

57ARG TIMESTAMP_UWSGI=0

58ARG TIMESTAMP_UWSGI=0

58ARG VERSION_GITCOMMIT=unknown

59ARG VERSION_GITCOMMIT=unknown

60−RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

61+RUN /usr/bin/python3 -m compileall -q searx; \

61 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

62 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

62 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

63 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

63 if [ ! -z $VERSION_GITCOMMIT ]; then\

64 if [ ! -z $VERSION_GITCOMMIT ]; then\

searx/autocomplete.py+6−6

@@ -24,6 +24,7 @@ from httpx import HTTPError

26from searx import settings

27+from searx.data import ENGINES_LANGUAGES

27from searx.network import get as http_get

28from searx.network import get as http_get

28from searx.exceptions import SearxEngineResponseException

29from searx.exceptions import SearxEngineResponseException

@@ -93,12 +94,11 @@ def google(query, lang):

94def startpage(query, lang):

95def startpage(query, lang):

95 # startpage autocompleter

96 # startpage autocompleter

97+ lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')

96− url = 'https://startpage.com/do/suggest?{query}'

98+ url = 'https://startpage.com/suggestions?{query}'

97−

98− resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')

99+ resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))

99− if len(resp) > 1:

100+ data = resp.json()

100− return resp

101+ return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

101− return []

102

103

104def swisscows(query, lang):

searx/engines/gigablast.py+24−13

@@ -5,10 +5,10 @@

5# pylint: disable=missing-function-docstring, invalid-name

7import re

8−from json import loads

8+from json import loads, JSONDecodeError

9from urllib.parse import urlencode

10−# from searx import logger

11from searx.network import get

10from searx.network import get

11+from searx.exceptions import SearxEngineResponseException

13# about

14about = {

@@ -22,6 +22,9 @@ about = {

23# engine dependent config

24categories = ['general']

25+collections = 'main'

26+search_type = ''

27+fast = 0

25# gigablast's pagination is totally damaged, don't use it

28# gigablast's pagination is totally damaged, don't use it

26paging = False

29paging = False

27safesearch = True

30safesearch = True

@@ -34,6 +37,8 @@ base_url = 'https://gigablast.com'

34extra_param = ''

37extra_param = ''

35extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'

38extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'

40+_wait_for_results_msg = 'Loading results takes too long. Please enable fast option in gigablast engine.'

41+

37def parse_extra_param(text):

42def parse_extra_param(text):

39 # example:

44 # example:

@@ -54,7 +59,6 @@ def parse_extra_param(text):

54 if re_var is not None and re_var.search(line):

59 if re_var is not None and re_var.search(line):

55 extra_param += re_var.search(line).group(1)

60 extra_param += re_var.search(line).group(1)

56 break

61 break

57− # logger.debug('gigablast extra_param="%s"', extra_param)

59def init(engine_settings=None): # pylint: disable=unused-argument

63def init(engine_settings=None): # pylint: disable=unused-argument

60 parse_extra_param(get(base_url + extra_param_path).text)

64 parse_extra_param(get(base_url + extra_param_path).text)

@@ -65,14 +69,17 @@ def request(query, params): # pylint: disable=unused-argument

66 # see API http://www.gigablast.com/api.html#/search

70 # see API http://www.gigablast.com/api.html#/search

67 # Take into account, that the API has some quirks ..

71 # Take into account, that the API has some quirks ..

68−

69− query_args = dict(

72+ query_args = {

70− c = 'main'

73+ 'c': collections,

71− , format = 'json'

74+ 'format': 'json',

72− , q = query

75+ 'q': query,

73− , dr = 1

76+ 'dr': 1 ,

74− , showgoodimages = 0

77+ 'showgoodimages': 0,

75− )

78+ 'fast': fast,

79+ }

80+

81+ if search_type != '':

82+ query_args['searchtype'] = search_type

77 if params['language'] and params['language'] != 'all':

84 if params['language'] and params['language'] != 'all':

78 query_args['qlangcountry'] = params['language']

85 query_args['qlangcountry'] = params['language']

@@ -90,9 +97,13 @@ def request(query, params): # pylint: disable=unused-argument

90def response(resp):

97def response(resp):

91 results = []

98 results = []

100+ try:

93− response_json = loads(resp.text)

101+ response_json = loads(resp.text)

102+ except JSONDecodeError as e:

103+ if 'Waiting for results' in resp.text:

104+ raise SearxEngineResponseException(message=_wait_for_results_msg) # pylint: disable=raise-missing-from

105+ raise e

106

95− # logger.debug('gigablast returns %s results', len(response_json['results']))

107

97 for result in response_json['results']:

108 for result in response_json['results']:

98 # see "Example JSON Output (&format=json)"

109 # see "Example JSON Output (&format=json)"

searx/network/__init__.py+22−1

@@ -5,6 +5,7 @@ import threading

5import concurrent.futures

6from time import time

7from queue import SimpleQueue

8+from types import MethodType

9import httpx

10import httpx

10import h2.exceptions

11import h2.exceptions

@@ -134,15 +135,27 @@ async def stream_chunk_to_queue(network, q, method, url, **kwargs):

134 try:

135 try:

135 async with await network.stream(method, url, **kwargs) as response:

136 async with await network.stream(method, url, **kwargs) as response:

136 q.put(response)

137 q.put(response)

138+ # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding

139+ # https://www.python-httpx.org/quickstart/#streaming-responses

137 async for chunk in response.aiter_bytes(65536):

140 async for chunk in response.aiter_bytes(65536):

138 if len(chunk) > 0:

141 if len(chunk) > 0:

139 q.put(chunk)

142 q.put(chunk)

143+ except httpx.ResponseClosed as e:

144+ # the response was closed

145+ pass

140 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

146 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

141 q.put(e)

147 q.put(e)

142 finally:

148 finally:

143 q.put(None)

149 q.put(None)

144

150

145

151

152+def _close_response_method(self):

153+ asyncio.run_coroutine_threadsafe(

154+ self.aclose(),

155+ get_loop()

156+ )

157+

158+

146def stream(method, url, **kwargs):

159def stream(method, url, **kwargs):

147 """Replace httpx.stream.

160 """Replace httpx.stream.

148

161

@@ -158,10 +171,18 @@ def stream(method, url, **kwargs):

158 q = SimpleQueue()

171 q = SimpleQueue()

159 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

172 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

160 get_loop())

173 get_loop())

174+ # yield response

175+ response = q.get()

176+ if isinstance(response, Exception):

177+ raise response

178+ response.close = MethodType(_close_response_method, response)

179+ yield response

180+

181+ # yield chunks

161 chunk_or_exception = q.get()

182 chunk_or_exception = q.get()

162 while chunk_or_exception is not None:

183 while chunk_or_exception is not None:

163 if isinstance(chunk_or_exception, Exception):

184 if isinstance(chunk_or_exception, Exception):

164 raise chunk_or_exception

185 raise chunk_or_exception

165 yield chunk_or_exception

186 yield chunk_or_exception

166 chunk_or_exception = q.get()

187 chunk_or_exception = q.get()

167− return future.result()

188+ future.result()

searx/network/network.py+8

@@ -326,6 +326,14 @@ def initialize(settings_engines=None, settings_outgoing=None):

326 if isinstance(network, str):

327 NETWORKS[engine_name] = NETWORKS[network]

328

329+ # the /image_proxy endpoint has a dedicated network.

330+ # same parameters than the default network, but HTTP/2 is disabled.

331+ # It decreases the CPU load average, and the total time is more or less the same

332+ if 'image_proxy' not in NETWORKS:

333+ image_proxy_params = default_params.copy()

334+ image_proxy_params['enable_http2'] = False

335+ NETWORKS['image_proxy'] = new_network(image_proxy_params)

336+

329

337

330@atexit.register

338@atexit.register

331def done():

339def done():

searx/settings.yml+12−8

@@ -505,6 +505,13 @@ engines:

505 engine : gigablast

506 shortcut : gb

507 timeout : 3.0

508+ # Collection names to search in, separated by whitespaces.

509+ # By default searx, queries the main collection.

510+ #collections: main

511+ # Search types, not required. possible values: images, news

512+ #search_type: images

513+ # Enable fast to get results quicker.

514+ #fast : 1

508 disabled: True

515 disabled: True

509 additional_tests:

516 additional_tests:

510 rosebud: *test_rosebud

517 rosebud: *test_rosebud

@@ -690,22 +697,19 @@ engines:

690

697

691 - name : hoogle

698 - name : hoogle

692 engine : xpath

699 engine : xpath

693− paging : True

694 search_url : https://hoogle.haskell.org/?hoogle={query}&start={pageno}

700 search_url : https://hoogle.haskell.org/?hoogle={query}&start={pageno}

695 results_xpath : '//div[@class="result"]'

701 results_xpath : '//div[@class="result"]'

696− title_xpath : './/div[@class="ans"]//a'

702+ title_xpath : './div[@class="ans"]'

697− url_xpath : './/div[@class="ans"]//a/@href'

703+ url_xpath : './div[@class="ans"]//a/@href'

698− content_xpath : './/div[@class="from"]'

704+ content_xpath : './div[contains(@class, "doc")]'

699− page_size : 20

700 categories : it

705 categories : it

701 shortcut : ho

706 shortcut : ho

702 about:

707 about:

703− website: https://www.haskell.org/

708+ website: https://hoogle.haskell.org/

704 wikidata_id: Q34010

709 wikidata_id: Q34010

705− official_api_documentation: https://hackage.haskell.org/api

706 use_official_api: false

710 use_official_api: false

707 require_api_key: false

711 require_api_key: false

708− results: JSON

712+ results: HTML

709

713

710 - name : imdb

714 - name : imdb

711 engine : imdb

715 engine : imdb

searx/utils.py+1−5

@@ -272,11 +272,7 @@ def dict_subset(d, properties):

272 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

273 {'A': 'a'}

274 """

275− result = {}

276− for k in properties:

275+ return {k: d[k] for k in properties if k in d}

277− if k in d:

278− result[k] = d[k]

279− return result

280

276

281

277

282def get_torrent_size(filesize, filesize_multiplier):

278def get_torrent_size(filesize, filesize_multiplier):

searx/webapp.py+41−13

@@ -94,7 +94,7 @@ from searx.plugins import plugins

94from searx.plugins.oa_doi_rewrite import get_doi_resolver

95from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

96from searx.answerers import answerers

97−from searx.network import stream as http_stream

97+from searx.network import stream as http_stream, set_context_network_name

98from searx.answerers import ask

99from searx.metrology.error_recorder import errors_per_engines

100from searx.settings_loader import get_default_settings_path

@@ -921,6 +921,8 @@ def _is_selected_language_supported(engine, preferences):

921

922@app.route('/image_proxy', methods=['GET'])

923def image_proxy():

924+ # pylint: disable=too-many-return-statements, too-many-branches

925+

924 url = request.args.get('url')

926 url = request.args.get('url')

925

927

926 if not url:

928 if not url:

@@ -932,14 +934,21 @@ def image_proxy():

932 return '', 400

934 return '', 400

933

935

934 maximum_size = 5 * 1024 * 1024

936 maximum_size = 5 * 1024 * 1024

935−

937+ forward_resp = False

938+ resp = None

936 try:

939 try:

937− headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})

940+ request_headers = {

938− headers['User-Agent'] = gen_useragent()

941+ 'User-Agent': gen_useragent(),

942+ 'Accept': 'image/webp,*/*',

943+ 'Accept-Encoding': 'gzip, deflate',

944+ 'Sec-GPC': '1',

945+ 'DNT': '1',

946+ }

947+ set_context_network_name('image_proxy')

939 stream = http_stream(

948 stream = http_stream(

940 method='GET',

949 method='GET',

941 url=url,

950 url=url,

942− headers=headers,

951+ headers=request_headers,

943 timeout=settings['outgoing']['request_timeout'],

952 timeout=settings['outgoing']['request_timeout'],

944 follow_redirects=True,

953 follow_redirects=True,

945 max_redirects=20)

954 max_redirects=20)

@@ -949,25 +958,37 @@ def image_proxy():

949 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

958 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

950 return 'Max size', 400

959 return 'Max size', 400

951

960

952− if resp.status_code == 304:

953− return '', resp.status_code

954−

955 if resp.status_code != 200:

961 if resp.status_code != 200:

956 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

962 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

957 if resp.status_code >= 400:

963 if resp.status_code >= 400:

958 return '', resp.status_code

964 return '', resp.status_code

959 return '', 400

965 return '', 400

960

966

961− if not resp.headers.get('content-type', '').startswith('image/'):

967+ if not resp.headers.get('Content-Type', '').startswith('image/'):

962− logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))

968+ logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', ''))

963 return '', 400

969 return '', 400

964

970

965− headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

971+ forward_resp = True

972+ except httpx.HTTPError:

973+ logger.exception('HTTP error')

974+ return '', 400

975+ finally:

976+ if resp and not forward_resp:

977+ # the code is about to return an HTTP 400 error to the browser

978+ # we make sure to close the response between searxng and the HTTP server

979+ try:

980+ resp.close()

981+ except httpx.HTTPError:

982+ logger.exception('HTTP error on closing')

966

983

967− total_length = 0

984+ try:

985+ headers = dict_subset(

986+ resp.headers,

987+ {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'}

988+ )

968

989

969 def forward_chunk():

990 def forward_chunk():

970− nonlocal total_length

991+ total_length = 0

971 for chunk in stream:

992 for chunk in stream:

972 total_length += len(chunk)

993 total_length += len(chunk)

973 if total_length > maximum_size:

994 if total_length > maximum_size:

@@ -1148,6 +1169,13 @@ def run():

1148 )

1169 )

1149

1170

1150

1171

1172+def patch_application(app):

1173+ # serve pages with HTTP/1.1

1174+ WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])

1175+ # patch app to handle non root url-s behind proxy & wsgi

1176+ app.wsgi_app = ReverseProxyPathFix(ProxyFix(app.wsgi_app))

1177+

1178+

1151class ReverseProxyPathFix:

1179class ReverseProxyPathFix:

1152 '''Wrap the application in this middleware and configure the

1180 '''Wrap the application in this middleware and configure the

1153 front-end server to add these headers, to let you quietly bind

1181 front-end server to add these headers, to let you quietly bind