on 0c351ea help sign in
8 files changed+48−117
since splitsnapshots
unifiedsplit
Dockerfile+2−3 searx/autocomplete.py+6−6 searx/engines/gigablast.py+13−24 searx/network/__init__.py+1−22 searx/network/network.py−8 searx/settings.yml+8−12 searx/utils.py+5−1 searx/webapp.py+13−41
Dockerfile+2−3
@@ -51,14 +51,13 @@ RUN apk upgrade --no-cache \
5151 && apk del build-dependencies \
5252 && rm -rf /root/.cache
5353
54−COPY searx ./searx
55−COPY dockerfiles ./dockerfiles
54+COPY --chown=searx:searx . .
5655
5756ARG TIMESTAMP_SETTINGS=0
5857ARG TIMESTAMP_UWSGI=0
5958ARG VERSION_GITCOMMIT=unknown
6059
61−RUN /usr/bin/python3 -m compileall -q searx; \
60+RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
6261 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \
6362 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \
6463 if [ ! -z $VERSION_GITCOMMIT ]; then\
searx/autocomplete.py+6−6
@@ -24,7 +24,6 @@ from httpx import HTTPError
2424
2525
2626from searx import settings
27−from searx.data import ENGINES_LANGUAGES
2827from searx.network import get as http_get
2928from searx.exceptions import SearxEngineResponseException
3029
@@ -94,11 +93,12 @@ def google(query, lang):
9493
9594def startpage(query, lang):
9695 # startpage autocompleter
97− lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')
98− url = 'https://startpage.com/suggestions?{query}'
99− resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))
100− data = resp.json()
101− return [e['text'] for e in data.get('suggestions', []) if 'text' in e]
96+ url = 'https://startpage.com/do/suggest?{query}'
97+
98+ resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')
99+ if len(resp) > 1:
100+ return resp
101+ return []
102102
103103
104104def swisscows(query, lang):
searx/engines/gigablast.py+13−24
@@ -5,10 +5,10 @@
55# pylint: disable=missing-function-docstring, invalid-name
66
77import re
8−from json import loads, JSONDecodeError
8+from json import loads
99from urllib.parse import urlencode
10+# from searx import logger
1011from searx.network import get
11−from searx.exceptions import SearxEngineResponseException
1212
1313# about
1414about = {
@@ -22,9 +22,6 @@ about = {
2222
2323# engine dependent config
2424categories = ['general']
25−collections = 'main'
26−search_type = ''
27−fast = 0
2825# gigablast's pagination is totally damaged, don't use it
2926paging = False
3027safesearch = True
@@ -37,8 +34,6 @@ base_url = 'https://gigablast.com'
3734extra_param = ''
3835extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'
3936
40−_wait_for_results_msg = 'Loading results takes too long. Please enable fast option in gigablast engine.'
41−
4237def parse_extra_param(text):
4338
4439 # example:
@@ -59,6 +54,7 @@ def parse_extra_param(text):
5954 if re_var is not None and re_var.search(line):
6055 extra_param += re_var.search(line).group(1)
6156 break
57+ # logger.debug('gigablast extra_param="%s"', extra_param)
6258
6359def init(engine_settings=None): # pylint: disable=unused-argument
6460 parse_extra_param(get(base_url + extra_param_path).text)
@@ -69,17 +65,14 @@ def request(query, params): # pylint: disable=unused-argument
6965
7066 # see API http://www.gigablast.com/api.html#/search
7167 # Take into account, that the API has some quirks ..
72− query_args = {
73− 'c': collections,
74− 'format': 'json',
75− 'q': query,
76− 'dr': 1 ,
77− 'showgoodimages': 0,
78− 'fast': fast,
79− }
80−
81− if search_type != '':
82− query_args['searchtype'] = search_type
68+
69+ query_args = dict(
70+ c = 'main'
71+ , format = 'json'
72+ , q = query
73+ , dr = 1
74+ , showgoodimages = 0
75+ )
8376
8477 if params['language'] and params['language'] != 'all':
8578 query_args['qlangcountry'] = params['language']
@@ -97,13 +90,9 @@ def request(query, params): # pylint: disable=unused-argument
9790def response(resp):
9891 results = []
9992
100− try:
101− response_json = loads(resp.text)
102− except JSONDecodeError as e:
103− if 'Waiting for results' in resp.text:
104− raise SearxEngineResponseException(message=_wait_for_results_msg) # pylint: disable=raise-missing-from
105− raise e
93+ response_json = loads(resp.text)
10694
95+ # logger.debug('gigablast returns %s results', len(response_json['results']))
10796
10897 for result in response_json['results']:
10998 # see "Example JSON Output (&format=json)"
searx/network/__init__.py+1−22
@@ -5,7 +5,6 @@ import threading
55import concurrent.futures
66from time import time
77from queue import SimpleQueue
8−from types import MethodType
98
109import httpx
1110import h2.exceptions
@@ -135,27 +134,15 @@ async def stream_chunk_to_queue(network, q, method, url, **kwargs):
135134 try:
136135 async with await network.stream(method, url, **kwargs) as response:
137136 q.put(response)
138− # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding
139− # https://www.python-httpx.org/quickstart/#streaming-responses
140137 async for chunk in response.aiter_bytes(65536):
141138 if len(chunk) > 0:
142139 q.put(chunk)
143− except httpx.ResponseClosed as e:
144− # the response was closed
145− pass
146140 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:
147141 q.put(e)
148142 finally:
149143 q.put(None)
150144
151145
152−def _close_response_method(self):
153− asyncio.run_coroutine_threadsafe(
154− self.aclose(),
155− get_loop()
156− )
157−
158−
159146def stream(method, url, **kwargs):
160147 """Replace httpx.stream.
161148
@@ -171,18 +158,10 @@ def stream(method, url, **kwargs):
171158 q = SimpleQueue()
172159 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),
173160 get_loop())
174− # yield response
175− response = q.get()
176− if isinstance(response, Exception):
177− raise response
178− response.close = MethodType(_close_response_method, response)
179− yield response
180−
181− # yield chunks
182161 chunk_or_exception = q.get()
183162 while chunk_or_exception is not None:
184163 if isinstance(chunk_or_exception, Exception):
185164 raise chunk_or_exception
186165 yield chunk_or_exception
187166 chunk_or_exception = q.get()
188− future.result()
167+ return future.result()
searx/network/network.py−8
@@ -326,14 +326,6 @@ def initialize(settings_engines=None, settings_outgoing=None):
326326 if isinstance(network, str):
327327 NETWORKS[engine_name] = NETWORKS[network]
328328
329− # the /image_proxy endpoint has a dedicated network.
330− # same parameters than the default network, but HTTP/2 is disabled.
331− # It decreases the CPU load average, and the total time is more or less the same
332− if 'image_proxy' not in NETWORKS:
333− image_proxy_params = default_params.copy()
334− image_proxy_params['enable_http2'] = False
335− NETWORKS['image_proxy'] = new_network(image_proxy_params)
336−
337329
338330@atexit.register
339331def done():
searx/settings.yml+8−12
@@ -505,13 +505,6 @@ engines:
505505 engine : gigablast
506506 shortcut : gb
507507 timeout : 3.0
508− # Collection names to search in, separated by whitespaces.
509− # By default searx, queries the main collection.
510− #collections: main
511− # Search types, not required. possible values: images, news
512− #search_type: images
513− # Enable fast to get results quicker.
514− #fast : 1
515508 disabled: True
516509 additional_tests:
517510 rosebud: *test_rosebud
@@ -697,19 +690,22 @@ engines:
697690
698691 - name : hoogle
699692 engine : xpath
693+ paging : True
700694 search_url : https://hoogle.haskell.org/?hoogle={query}&start={pageno}
701695 results_xpath : '//div[@class="result"]'
702− title_xpath : './div[@class="ans"]'
703− url_xpath : './div[@class="ans"]//a/@href'
704− content_xpath : './div[contains(@class, "doc")]'
696+ title_xpath : './/div[@class="ans"]//a'
697+ url_xpath : './/div[@class="ans"]//a/@href'
698+ content_xpath : './/div[@class="from"]'
699+ page_size : 20
705700 categories : it
706701 shortcut : ho
707702 about:
708− website: https://hoogle.haskell.org/
703+ website: https://www.haskell.org/
709704 wikidata_id: Q34010
705+ official_api_documentation: https://hackage.haskell.org/api
710706 use_official_api: false
711707 require_api_key: false
712− results: HTML
708+ results: JSON
713709
714710 - name : imdb
715711 engine : imdb
searx/utils.py+5−1
@@ -272,7 +272,11 @@ def dict_subset(d, properties):
272272 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])
273273 {'A': 'a'}
274274 """
275− return {k: d[k] for k in properties if k in d}
275+ result = {}
276+ for k in properties:
277+ if k in d:
278+ result[k] = d[k]
279+ return result
276280
277281
278282def get_torrent_size(filesize, filesize_multiplier):
searx/webapp.py+13−41
@@ -94,7 +94,7 @@ from searx.plugins import plugins
9494from searx.plugins.oa_doi_rewrite import get_doi_resolver
9595from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
9696from searx.answerers import answerers
97−from searx.network import stream as http_stream, set_context_network_name
97+from searx.network import stream as http_stream
9898from searx.answerers import ask
9999from searx.metrology.error_recorder import errors_per_engines
100100from searx.settings_loader import get_default_settings_path
@@ -921,8 +921,6 @@ def _is_selected_language_supported(engine, preferences):
921921
922922@app.route('/image_proxy', methods=['GET'])
923923def image_proxy():
924− # pylint: disable=too-many-return-statements, too-many-branches
925−
926924 url = request.args.get('url')
927925
928926 if not url:
@@ -934,21 +932,14 @@ def image_proxy():
934932 return '', 400
935933
936934 maximum_size = 5 * 1024 * 1024
937− forward_resp = False
938− resp = None
935+
939936 try:
940− request_headers = {
941− 'User-Agent': gen_useragent(),
942− 'Accept': 'image/webp,*/*',
943− 'Accept-Encoding': 'gzip, deflate',
944− 'Sec-GPC': '1',
945− 'DNT': '1',
946− }
947− set_context_network_name('image_proxy')
937+ headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})
938+ headers['User-Agent'] = gen_useragent()
948939 stream = http_stream(
949940 method='GET',
950941 url=url,
951− headers=request_headers,
942+ headers=headers,
952943 timeout=settings['outgoing']['request_timeout'],
953944 follow_redirects=True,
954945 max_redirects=20)
@@ -958,37 +949,25 @@ def image_proxy():
958949 if content_length and content_length.isdigit() and int(content_length) > maximum_size:
959950 return 'Max size', 400
960951
952+ if resp.status_code == 304:
953+ return '', resp.status_code
954+
961955 if resp.status_code != 200:
962956 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))
963957 if resp.status_code >= 400:
964958 return '', resp.status_code
965959 return '', 400
966960
967− if not resp.headers.get('Content-Type', '').startswith('image/'):
968− logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', ''))
961+ if not resp.headers.get('content-type', '').startswith('image/'):
962+ logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
969963 return '', 400
970964
971− forward_resp = True
972− except httpx.HTTPError:
973− logger.exception('HTTP error')
974− return '', 400
975− finally:
976− if resp and not forward_resp:
977− # the code is about to return an HTTP 400 error to the browser
978− # we make sure to close the response between searxng and the HTTP server
979− try:
980− resp.close()
981− except httpx.HTTPError:
982− logger.exception('HTTP error on closing')
965+ headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})
983966
984− try:
985− headers = dict_subset(
986− resp.headers,
987− {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'}
988− )
967+ total_length = 0
989968
990969 def forward_chunk():
991− total_length = 0
970+ nonlocal total_length
992971 for chunk in stream:
993972 total_length += len(chunk)
994973 if total_length > maximum_size:
@@ -1169,13 +1148,6 @@ def run():
11691148 )
11701149
11711150
1172−def patch_application(app):
1173− # serve pages with HTTP/1.1
1174− WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])
1175− # patch app to handle non root url-s behind proxy & wsgi
1176− app.wsgi_app = ReverseProxyPathFix(ProxyFix(app.wsgi_app))
1177−
1178−
11791151class ReverseProxyPathFix:
11801152 '''Wrap the application in this middleware and configure the
11811153 front-end server to add these headers, to let you quietly bind