036d80e...master · searx

Dockerfile+3−2

since splitsnapshots

unifiedsplit

@@ -51,13 +51,14 @@ RUN apk upgrade --no-cache \

51 && apk del build-dependencies \

52 && rm -rf /root/.cache

53

54−COPY --chown=searx:searx . .

54+COPY searx ./searx

55+COPY dockerfiles ./dockerfiles

55

56

56ARG TIMESTAMP_SETTINGS=0

57ARG TIMESTAMP_SETTINGS=0

57ARG TIMESTAMP_UWSGI=0

58ARG TIMESTAMP_UWSGI=0

58ARG VERSION_GITCOMMIT=unknown

59ARG VERSION_GITCOMMIT=unknown

59

60

60−RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

61+RUN /usr/bin/python3 -m compileall -q searx; \

61 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

62 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

62 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

63 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

63 if [ ! -z $VERSION_GITCOMMIT ]; then\

64 if [ ! -z $VERSION_GITCOMMIT ]; then\

searx/autocomplete.py+6−6

@@ -24,6 +24,7 @@ from httpx import HTTPError

24

25

26from searx import settings

27+from searx.data import ENGINES_LANGUAGES

27from searx.network import get as http_get

28from searx.network import get as http_get

28from searx.exceptions import SearxEngineResponseException

29from searx.exceptions import SearxEngineResponseException

29

30

@@ -93,12 +94,11 @@ def google(query, lang):

93

94

94def startpage(query, lang):

95def startpage(query, lang):

95 # startpage autocompleter

96 # startpage autocompleter

97+ lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')

96− url = 'https://startpage.com/do/suggest?{query}'

98+ url = 'https://startpage.com/suggestions?{query}'

97−

98− resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')

99+ resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))

99− if len(resp) > 1:

100+ data = resp.json()

100− return resp

101+ return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

101− return []

102

103

104def swisscows(query, lang):

searx/engines/gigablast.py+24−13

@@ -5,10 +5,10 @@

5# pylint: disable=missing-function-docstring, invalid-name

6

7import re

8−from json import loads

8+from json import loads, JSONDecodeError

9from urllib.parse import urlencode

10−# from searx import logger

11from searx.network import get

10from searx.network import get

11+from searx.exceptions import SearxEngineResponseException

12

13# about

14about = {

@@ -22,6 +22,9 @@ about = {

22

23# engine dependent config

24categories = ['general']

25+collections = 'main'

26+search_type = ''

27+fast = 0

25# gigablast's pagination is totally damaged, don't use it

28# gigablast's pagination is totally damaged, don't use it

26paging = False

29paging = False

27safesearch = True

30safesearch = True

@@ -34,6 +37,8 @@ base_url = 'https://gigablast.com'

34extra_param = ''

37extra_param = ''

35extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'

38extra_param_path='/search?c=main&qlangcountry=en-us&q=south&s=10'

36

39

40+_wait_for_results_msg = 'Loading results takes too long. Please enable fast option in gigablast engine.'

41+

37def parse_extra_param(text):

42def parse_extra_param(text):

38

43

39 # example:

44 # example:

@@ -54,7 +59,6 @@ def parse_extra_param(text):

54 if re_var is not None and re_var.search(line):

59 if re_var is not None and re_var.search(line):

55 extra_param += re_var.search(line).group(1)

60 extra_param += re_var.search(line).group(1)

56 break

61 break

57− # logger.debug('gigablast extra_param="%s"', extra_param)

58

62

59def init(engine_settings=None): # pylint: disable=unused-argument

63def init(engine_settings=None): # pylint: disable=unused-argument

60 parse_extra_param(get(base_url + extra_param_path).text)

64 parse_extra_param(get(base_url + extra_param_path).text)

@@ -65,14 +69,17 @@ def request(query, params): # pylint: disable=unused-argument

65

69

66 # see API http://www.gigablast.com/api.html#/search

70 # see API http://www.gigablast.com/api.html#/search

67 # Take into account, that the API has some quirks ..

71 # Take into account, that the API has some quirks ..

68−

69− query_args = dict(

72+ query_args = {

70− c = 'main'

73+ 'c': collections,

71− , format = 'json'

74+ 'format': 'json',

72− , q = query

75+ 'q': query,

73− , dr = 1

76+ 'dr': 1 ,

74− , showgoodimages = 0

77+ 'showgoodimages': 0,

75− )

78+ 'fast': fast,

79+ }

80+

81+ if search_type != '':

82+ query_args['searchtype'] = search_type

76

83

77 if params['language'] and params['language'] != 'all':

84 if params['language'] and params['language'] != 'all':

78 query_args['qlangcountry'] = params['language']

85 query_args['qlangcountry'] = params['language']

@@ -90,9 +97,13 @@ def request(query, params): # pylint: disable=unused-argument

90def response(resp):

97def response(resp):

91 results = []

98 results = []

92

99

100+ try:

93− response_json = loads(resp.text)

101+ response_json = loads(resp.text)

102+ except JSONDecodeError as e:

103+ if 'Waiting for results' in resp.text:

104+ raise SearxEngineResponseException(message=_wait_for_results_msg) # pylint: disable=raise-missing-from

105+ raise e

94

106

95− # logger.debug('gigablast returns %s results', len(response_json['results']))

96

107

97 for result in response_json['results']:

108 for result in response_json['results']:

98 # see "Example JSON Output (&format=json)"

109 # see "Example JSON Output (&format=json)"

searx/engines/google.py+1−1

@@ -8,7 +8,7 @@ Definitions`_.

8 https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions

9"""

10

11−# pylint: disable=invalid-name, missing-function-docstring

11+# pylint: disable=invalid-name, missing-function-docstring, too-many-branches

12

13from urllib.parse import urlencode

14from lxml import html

searx/engines/startpage.py+4−2

@@ -16,13 +16,15 @@ from lxml import html

16from babel import Locale

17from babel.localedata import locale_identifiers

18

19−from searx import network

19+from searx import logger

20+from searx.network import get

20from searx.utils import extract_text, eval_xpath, match_language

21from searx.utils import extract_text, eval_xpath, match_language

21from searx.exceptions import (

22from searx.exceptions import (

22 SearxEngineResponseException,

23 SearxEngineResponseException,

23 SearxEngineCaptchaException,

24 SearxEngineCaptchaException,

24)

25)

25

26

27+logger = logger.getChild('startpage')

26

28

27# about

29# about

28about = {

30about = {

@@ -84,7 +86,7 @@ def get_sc_code(headers):

84 if time() > (sc_code_ts + 3000):

86 if time() > (sc_code_ts + 3000):

85 logger.debug("query new sc time-stamp ...")

87 logger.debug("query new sc time-stamp ...")

86

88

87− resp = network.get(base_url, headers=headers)

89+ resp = get(base_url, headers=headers)

88 raise_captcha(resp)

90 raise_captcha(resp)

89 dom = html.fromstring(resp.text)

91 dom = html.fromstring(resp.text)

90

92

searx/engines/tineye.pynew file+86

@@ -0,0 +1,86 @@

1+"""

2+Tineye - Reverse search images

3+"""

4+

5+from json import loads

6+from urllib.parse import urlencode

7+

8+from datetime import datetime

9+

10+about = {

11+ "website": "https://tineye.com",

12+ "wikidata_id": "Q2382535",

13+ "use_official_api": False,

14+ "require_api_key": False,

15+ "results": "JSON",

16+}

17+

18+

19+categories = ['images']

20+paging = True

21+

22+safesearch = False

23+

24+

25+base_url = 'https://tineye.com'

26+search_string = '/result_json/?page={page}&{query}'

27+

28+

29+def request(query, params):

30+ params['url'] = base_url +\

31+ search_string.format(

32+ query=urlencode({'url': query}),

33+ page=params['pageno'])

34+

35+ params['headers'].update({

36+ 'Connection': 'keep-alive',

37+ 'Accept-Encoding': 'gzip, defalte, br',

38+ 'Host': 'tineye.com',

39+ 'DNT': '1',

40+ 'TE': 'trailers',

41+ })

42+

43+ return params

44+

45+

46+def response(resp):

47+ results = []

48+ # Define wanted results

49+ json_data = loads(resp.text)

50+ number_of_results = json_data['num_matches']

51+

52+ for i in json_data['matches']:

53+ for i in json_data['matches']:

54+ image_format = i['format']

55+ width = i['width']

56+ height = i['height']

57+ thumbnail_src = i['image_url']

58+ backlink = i['domains'][0]['backlinks'][0]

59+

60+ url = backlink['backlink']

61+ source = backlink['url']

62+ title = backlink['image_name']

63+ img_src = backlink['url']

64+

65+ # Get and convert published date

66+ api_date = backlink['crawl_date'][:-3]

67+ publishedDate = datetime.fromisoformat(api_date)

68+

69+ # Append results

70+ results.append({

71+ 'template': 'images.html',

72+ 'url': url,

73+ 'thumbnail_src': thumbnail_src,

74+ 'source': source,

75+ 'title': title,

76+ 'img_src': img_src,

77+ 'format': image_format,

78+ 'widht': width,

79+ 'height': height,

80+ 'publishedDate': publishedDate,

81+ })

82+

83+ # Append number of results

84+ results.append({'number_of_results': number_of_results})

85+

86+ return results

searx/network/__init__.py+22−1

@@ -5,6 +5,7 @@ import threading

5import concurrent.futures

6from time import time

7from queue import SimpleQueue

8+from types import MethodType

8

9

9import httpx

10import httpx

10import h2.exceptions

11import h2.exceptions

@@ -134,15 +135,27 @@ async def stream_chunk_to_queue(network, q, method, url, **kwargs):

134 try:

135 try:

135 async with await network.stream(method, url, **kwargs) as response:

136 async with await network.stream(method, url, **kwargs) as response:

136 q.put(response)

137 q.put(response)

138+ # aiter_raw: access the raw bytes on the response without applying any HTTP content decoding

139+ # https://www.python-httpx.org/quickstart/#streaming-responses

137 async for chunk in response.aiter_bytes(65536):

140 async for chunk in response.aiter_bytes(65536):

138 if len(chunk) > 0:

141 if len(chunk) > 0:

139 q.put(chunk)

142 q.put(chunk)

143+ except httpx.ResponseClosed as e:

144+ # the response was closed

145+ pass

140 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

146 except (httpx.HTTPError, OSError, h2.exceptions.ProtocolError) as e:

141 q.put(e)

147 q.put(e)

142 finally:

148 finally:

143 q.put(None)

149 q.put(None)

144

150

145

151

152+def _close_response_method(self):

153+ asyncio.run_coroutine_threadsafe(

154+ self.aclose(),

155+ get_loop()

156+ )

157+

158+

146def stream(method, url, **kwargs):

159def stream(method, url, **kwargs):

147 """Replace httpx.stream.

160 """Replace httpx.stream.

148

161

@@ -158,10 +171,18 @@ def stream(method, url, **kwargs):

158 q = SimpleQueue()

171 q = SimpleQueue()

159 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

172 future = asyncio.run_coroutine_threadsafe(stream_chunk_to_queue(get_network(), q, method, url, **kwargs),

160 get_loop())

173 get_loop())

174+ # yield response

175+ response = q.get()

176+ if isinstance(response, Exception):

177+ raise response

178+ response.close = MethodType(_close_response_method, response)

179+ yield response

180+

181+ # yield chunks

161 chunk_or_exception = q.get()

182 chunk_or_exception = q.get()

162 while chunk_or_exception is not None:

183 while chunk_or_exception is not None:

163 if isinstance(chunk_or_exception, Exception):

184 if isinstance(chunk_or_exception, Exception):

164 raise chunk_or_exception

185 raise chunk_or_exception

165 yield chunk_or_exception

186 yield chunk_or_exception

166 chunk_or_exception = q.get()

187 chunk_or_exception = q.get()

167− return future.result()

188+ future.result()

searx/network/network.py+8

@@ -326,6 +326,14 @@ def initialize(settings_engines=None, settings_outgoing=None):

326 if isinstance(network, str):

327 NETWORKS[engine_name] = NETWORKS[network]

328

329+ # the /image_proxy endpoint has a dedicated network.

330+ # same parameters than the default network, but HTTP/2 is disabled.

331+ # It decreases the CPU load average, and the total time is more or less the same

332+ if 'image_proxy' not in NETWORKS:

333+ image_proxy_params = default_params.copy()

334+ image_proxy_params['enable_http2'] = False

335+ NETWORKS['image_proxy'] = new_network(image_proxy_params)

336+

329

337

330@atexit.register

338@atexit.register

331def done():

339def done():

searx/results.py+8−2

@@ -6,6 +6,7 @@ from urllib.parse import urlparse, unquote

6from searx import logger

7from searx.engines import engines

8from searx.metrology.error_recorder import record_error

9+from searx.utils import add_scheme_to_url

9from searx import settings

10from searx import settings

10

11

12

@@ -240,10 +241,15 @@ class ResultContainer:

240 result['parsed_url'] = urlparse(result['url'])

241 result['parsed_url'] = urlparse(result['url'])

241

242

242 # if the result has no scheme, use http as default

243 # if the result has no scheme, use http as default

243− if not result['parsed_url'].scheme:

244+ if not result['parsed_url'].scheme or result['parsed_url'].scheme == '':

244− result['parsed_url'] = result['parsed_url']._replace(scheme="http")

245+ result['parsed_url'] = result['parsed_url']._replace(scheme='http')

245 result['url'] = result['parsed_url'].geturl()

246 result['url'] = result['parsed_url'].geturl()

246

247

248+ if 'thumbnail_src' in result:

249+ result['thumbnail_src'] = add_scheme_to_url(result['thumbnail_src'])

250+ if 'img_src' in result:

251+ result['img_src'] = add_scheme_to_url(result['img_src'])

252+

247 result['engines'] = set([result['engine']])

253 result['engines'] = set([result['engine']])

248

254

249 # strip multiple spaces and cariage returns from content

255 # strip multiple spaces and cariage returns from content

searx/settings.yml+19−8

@@ -410,6 +410,13 @@ engines:

410 timeout: 3.0

411 disabled : True

412

413+ - name : tineye

414+ engine : tineye

415+ shortcut : tin

416+ paging : True

417+ timeout : 9.0

418+ disabled : True

419+

413 - name : etools

420 - name : etools

414 engine : etools

421 engine : etools

415 shortcut : eto

422 shortcut : eto

@@ -498,6 +505,13 @@ engines:

498 engine : gigablast

505 engine : gigablast

499 shortcut : gb

506 shortcut : gb

500 timeout : 3.0

507 timeout : 3.0

508+ # Collection names to search in, separated by whitespaces.

509+ # By default searx, queries the main collection.

510+ #collections: main

511+ # Search types, not required. possible values: images, news

512+ #search_type: images

513+ # Enable fast to get results quicker.

514+ #fast : 1

501 disabled: True

515 disabled: True

502 additional_tests:

516 additional_tests:

503 rosebud: *test_rosebud

517 rosebud: *test_rosebud

@@ -683,22 +697,19 @@ engines:

683

697

684 - name : hoogle

698 - name : hoogle

685 engine : xpath

699 engine : xpath

686− paging : True

687 search_url : https://hoogle.haskell.org/?hoogle={query}&start={pageno}

700 search_url : https://hoogle.haskell.org/?hoogle={query}&start={pageno}

688 results_xpath : '//div[@class="result"]'

701 results_xpath : '//div[@class="result"]'

689− title_xpath : './/div[@class="ans"]//a'

702+ title_xpath : './div[@class="ans"]'

690− url_xpath : './/div[@class="ans"]//a/@href'

703+ url_xpath : './div[@class="ans"]//a/@href'

691− content_xpath : './/div[@class="from"]'

704+ content_xpath : './div[contains(@class, "doc")]'

692− page_size : 20

693 categories : it

705 categories : it

694 shortcut : ho

706 shortcut : ho

695 about:

707 about:

696− website: https://www.haskell.org/

708+ website: https://hoogle.haskell.org/

697 wikidata_id: Q34010

709 wikidata_id: Q34010

698− official_api_documentation: https://hackage.haskell.org/api

699 use_official_api: false

710 use_official_api: false

700 require_api_key: false

711 require_api_key: false

701− results: JSON

712+ results: HTML

702

713

703 - name : imdb

714 - name : imdb

704 engine : imdb

715 engine : imdb

searx/utils.py+12−6

@@ -7,7 +7,7 @@ from numbers import Number

7from os.path import splitext, join

8from random import choice

9from html.parser import HTMLParser

10−from urllib.parse import urljoin, urlparse

10+from urllib.parse import urljoin, urlparse, urlunparse

11

12from lxml import html

13from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult

@@ -214,6 +214,16 @@ def normalize_url(url, base_url):

214 return url

215

216

217+def add_scheme_to_url(url, scheme="https"):

218+ """Add schema to URL: if scheme is missing from the URL, then add it."""

219+

220+ parsed = urlparse(url)

221+ if parsed.scheme == '':

222+ parsed_with_scheme = parsed._replace(scheme=scheme)

223+ return urlunparse(parsed_with_scheme)

224+ return url

225+

226+

217def extract_url(xpath_results, base_url):

227def extract_url(xpath_results, base_url):

218 """Extract and normalize URL from lxml Element

228 """Extract and normalize URL from lxml Element

219

229

@@ -262,11 +272,7 @@ def dict_subset(d, properties):

262 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

272 >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

263 {'A': 'a'}

273 {'A': 'a'}

264 """

274 """

265− result = {}

266− for k in properties:

275+ return {k: d[k] for k in properties if k in d}

267− if k in d:

268− result[k] = d[k]

269− return result

270

276

271

277

272def get_torrent_size(filesize, filesize_multiplier):

278def get_torrent_size(filesize, filesize_multiplier):

searx/webapp.py+41−13

@@ -94,7 +94,7 @@ from searx.plugins import plugins

94from searx.plugins.oa_doi_rewrite import get_doi_resolver

95from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

96from searx.answerers import answerers

97−from searx.network import stream as http_stream

97+from searx.network import stream as http_stream, set_context_network_name

98from searx.answerers import ask

99from searx.metrology.error_recorder import errors_per_engines

100from searx.settings_loader import get_default_settings_path

@@ -921,6 +921,8 @@ def _is_selected_language_supported(engine, preferences):

921

922@app.route('/image_proxy', methods=['GET'])

923def image_proxy():

924+ # pylint: disable=too-many-return-statements, too-many-branches

925+

924 url = request.args.get('url')

926 url = request.args.get('url')

925

927

926 if not url:

928 if not url:

@@ -932,14 +934,21 @@ def image_proxy():

932 return '', 400

934 return '', 400

933

935

934 maximum_size = 5 * 1024 * 1024

936 maximum_size = 5 * 1024 * 1024

935−

937+ forward_resp = False

938+ resp = None

936 try:

939 try:

937− headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})

940+ request_headers = {

938− headers['User-Agent'] = gen_useragent()

941+ 'User-Agent': gen_useragent(),

942+ 'Accept': 'image/webp,*/*',

943+ 'Accept-Encoding': 'gzip, deflate',

944+ 'Sec-GPC': '1',

945+ 'DNT': '1',

946+ }

947+ set_context_network_name('image_proxy')

939 stream = http_stream(

948 stream = http_stream(

940 method='GET',

949 method='GET',

941 url=url,

950 url=url,

942− headers=headers,

951+ headers=request_headers,

943 timeout=settings['outgoing']['request_timeout'],

952 timeout=settings['outgoing']['request_timeout'],

944 follow_redirects=True,

953 follow_redirects=True,

945 max_redirects=20)

954 max_redirects=20)

@@ -949,25 +958,37 @@ def image_proxy():

949 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

958 if content_length and content_length.isdigit() and int(content_length) > maximum_size:

950 return 'Max size', 400

959 return 'Max size', 400

951

960

952− if resp.status_code == 304:

953− return '', resp.status_code

954−

955 if resp.status_code != 200:

961 if resp.status_code != 200:

956 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

962 logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))

957 if resp.status_code >= 400:

963 if resp.status_code >= 400:

958 return '', resp.status_code

964 return '', resp.status_code

959 return '', 400

965 return '', 400

960

966

961− if not resp.headers.get('content-type', '').startswith('image/'):

967+ if not resp.headers.get('Content-Type', '').startswith('image/'):

962− logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))

968+ logger.debug('image-proxy: wrong content-type: %s', resp.headers.get('Content-Type', ''))

963 return '', 400

969 return '', 400

964

970

965− headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

971+ forward_resp = True

972+ except httpx.HTTPError:

973+ logger.exception('HTTP error')

974+ return '', 400

975+ finally:

976+ if resp and not forward_resp:

977+ # the code is about to return an HTTP 400 error to the browser

978+ # we make sure to close the response between searxng and the HTTP server

979+ try:

980+ resp.close()

981+ except httpx.HTTPError:

982+ logger.exception('HTTP error on closing')

966

983

967− total_length = 0

984+ try:

985+ headers = dict_subset(

986+ resp.headers,

987+ {'Content-Type', 'Content-Encoding', 'Content-Length', 'Length'}

988+ )

968

989

969 def forward_chunk():

990 def forward_chunk():

970− nonlocal total_length

991+ total_length = 0

971 for chunk in stream:

992 for chunk in stream:

972 total_length += len(chunk)

993 total_length += len(chunk)

973 if total_length > maximum_size:

994 if total_length > maximum_size:

@@ -1148,6 +1169,13 @@ def run():

1148 )

1169 )

1149

1170

1150

1171

1172+def patch_application(app):

1173+ # serve pages with HTTP/1.1

1174+ WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server']['http_protocol_version'])

1175+ # patch app to handle non root url-s behind proxy & wsgi

1176+ app.wsgi_app = ReverseProxyPathFix(ProxyFix(app.wsgi_app))

1177+

1178+

1151class ReverseProxyPathFix:

1179class ReverseProxyPathFix:

1152 '''Wrap the application in this middleware and configure the

1180 '''Wrap the application in this middleware and configure the

1153 front-end server to add these headers, to let you quietly bind

1181 front-end server to add these headers, to let you quietly bind