master..3a0f896 · searx

git.iko.soy/searx

121 files changed+8298−37126

unifiedsplit

.github/workflows/data-update.yml+2−2

@@ -29,7 +29,7 @@ jobs:

2929 - name: Set up Python

3030 uses: actions/setup-python@v2

3131 with:

32− python-version: '3.10'

32+ python-version: '3.9'

3333 architecture: 'x64'

3434

3535 - name: Install Python dependencies

@@ -40,7 +40,7 @@ jobs:

4040 env:

4141 FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}

4242 run: |

43− V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"

43+ V=1 manage pyenv.cmd python "$FETCH_SCRIPT"

4444

4545 - name: Create Pull Request

4646 id: cpr

.github/workflows/integration.yml+3−32

@@ -13,7 +13,7 @@ jobs:

1313 strategy:

1414 matrix:

1515 os: [ubuntu-20.04]

16− python-version: [3.7, 3.8, 3.9, "3.10"]

16+ python-version: [3.6, 3.7, 3.8, 3.9]

1717 steps:

1818 - name: Checkout

1919 uses: actions/checkout@v2

@@ -56,17 +56,6 @@ jobs:

5656 uses: actions/checkout@v2

5757 - name: Install Ubuntu packages

5858 run: sudo ./utils/searx.sh install packages

59− - name: Set up Python

60− uses: actions/setup-python@v2

61− with:

62− python-version: '3.9'

63− architecture: 'x64'

64− - name: Cache Python dependencies

65− id: cache-python

66− uses: actions/cache@v2

67− with:

68− path: ./local

69− key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

7059 - name: Install node dependencies

7160 run: make V=1 node.env

7261 - name: Build themes

@@ -79,21 +68,14 @@ jobs:

7968 - name: Checkout

8069 uses: actions/checkout@v2

8170 with:

82− fetch-depth: '0'

8371 persist-credentials: false

8472 - name: Install Ubuntu packages

8573 run: sudo ./utils/searx.sh install buildhost

8674 - name: Set up Python

8775 uses: actions/setup-python@v2

8876 with:

89− python-version: '3.10'

77+ python-version: '3.9'

9078 architecture: 'x64'

91− - name: Cache Python dependencies

92− id: cache-python

93− uses: actions/cache@v2

94− with:

95− path: ./local

96− key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

9779 - name: Build documentation

9880 run: |

9981 make V=1 docs.clean docs.html

@@ -115,7 +97,7 @@ jobs:

11597 - documentation

11698 env:

11799 DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

118− runs-on: ubuntu-20.04

100+ runs-on: ubuntu-18.04

119101 steps:

120102 - name: Checkout

121103 if: env.DOCKERHUB_USERNAME != null

@@ -123,17 +105,6 @@ jobs:

123105 with:

124106 # make sure "make docker.push" can get the git history

125107 fetch-depth: '0'

126− - name: Set up Python

127− uses: actions/setup-python@v2

128− with:

129− python-version: '3.9'

130− architecture: 'x64'

131− - name: Cache Python dependencies

132− id: cache-python

133− uses: actions/cache@v2

134− with:

135− path: ./local

136− key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

137108 - name: Set up QEMU

138109 if: env.DOCKERHUB_USERNAME != null

139110 uses: docker/setup-qemu-action@v1

.gitignore−2

@@ -26,5 +26,3 @@ dist/

2626local/

2727gh-pages/

2828searx.egg-info/

29−.env

30−geckodriver.log

.pylintrc+1−1

@@ -59,7 +59,7 @@ confidence=

5959# --enable=similarities". If you want to run only the classes checker, but have

6060# no Warning level messages displayed, use"--disable=all --enable=classes

6161# --disable=W"

62−disable=bad-whitespace, duplicate-code, consider-using-f-string

62+disable=bad-whitespace, duplicate-code

6363

6464# Enable the message, report, category or checker with the given id(s). You can

6565# either give multiple identifier separated by comma (,) or put this option

AUTHORS.rst−1

@@ -165,4 +165,3 @@ generally made searx better:

165165- @jhigginbotham

166166- @xenrox

167167- @OliveiraHermogenes

168−- Paul Alcock @Guilvareux

Dockerfile+4−5

@@ -1,4 +1,4 @@

1−FROM alpine:3.15

1+FROM alpine:3.12

22ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

33EXPOSE 8080

44VOLUME /etc/searx

@@ -46,19 +46,18 @@ RUN apk upgrade --no-cache \

4646 uwsgi \

4747 uwsgi-python3 \

4848 brotli \

49− && pip3 install --upgrade pip wheel setuptools \

49+ && pip3 install --upgrade pip \

5050 && pip3 install --no-cache -r requirements.txt \

5151 && apk del build-dependencies \

5252 && rm -rf /root/.cache

5353

54−COPY searx ./searx

55−COPY dockerfiles ./dockerfiles

54+COPY --chown=searx:searx . .

5655

5756ARG TIMESTAMP_SETTINGS=0

5857ARG TIMESTAMP_UWSGI=0

5958ARG VERSION_GITCOMMIT=unknown

6059

61−RUN /usr/bin/python3 -m compileall -q searx; \

60+RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

6261 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

6362 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

6463 if [ ! -z $VERSION_GITCOMMIT ]; then\

README.rst+2−2

@@ -17,7 +17,7 @@

1717|OpenCollective searx backers|

1818|OpenCollective searx sponsors|

1919

20−Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **sɜːks**.

20+Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **səːks**.

2121

2222.. _metasearch engine: https://en.wikipedia.org/wiki/Metasearch_engine

2323

@@ -61,7 +61,7 @@ our homepage_.

6161.. _homepage: https://searx.github.io/searx

6262

6363contact:

64− openhub_ // twitter_ // IRC: #searx @ Libera (irc.libera.chat)

64+ openhub_ // twitter_ // IRC: #searx @ freenode

6565

6666.. _openhub: https://www.openhub.net/p/searx

6767.. _twitter: https://twitter.com/Searx_engine

docs/admin/settings.rst+13−51

@@ -130,12 +130,14 @@ Global Settings

130130 request_timeout : 2.0 # default timeout in seconds, can be override by engine

131131 # max_request_timeout: 10.0 # the maximum timeout in seconds

132132 useragent_suffix : "" # informations like an email address to the administrator

133− pool_connections : 100 # Maximum number of allowable connections, or None for no limits. The default is 100.

134− pool_maxsize : 10 # Number of allowable keep-alive connections, or None to always allow. The default is 10.

135− enable_http2: True # See https://www.python-httpx.org/http2/

133+ pool_connections : 100 # Number of different hosts

134+ pool_maxsize : 10 # Number of simultaneous requests by host

136135 # uncomment below section if you want to use a proxy

137136 # proxies:

138− # all://:

137+ # http:

138+ # - http://proxy1:8080

139+ # - http://proxy2:8080

140+ # https:

139141 # - http://proxy1:8080

140142 # - http://proxy2:8080

141143 # uncomment below section only if you have more than one network interface

@@ -143,7 +145,6 @@ Global Settings

143145 # source_ips:

144146 # - 1.1.1.1

145147 # - 1.1.1.2

146− # - fe80::/126

147148

148149

149150``request_timeout`` :

@@ -156,46 +157,20 @@ Global Settings

156157 Suffix to the user-agent searx uses to send requests to others engines. If an

157158 engine wish to block you, a contact info here may be useful to avoid that.

158159

159−``keepalive_expiry``:

160− Number of seconds to keep a connection in the pool. By default 5.0 seconds.

161−

162−.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying

160+.. _requests proxies: https://requests.readthedocs.io/en/latest/user/advanced/#proxies

161+.. _PySocks: https://pypi.org/project/PySocks/

163162

164163``proxies`` :

165− Define one or more proxies you wish to use, see `httpx proxies`_.

164+ Define one or more proxies you wish to use, see `requests proxies`_.

166165 If there are more than one proxy for one protocol (http, https),

167166 requests to the engines are distributed in a round-robin fashion.

168167

168+ - Proxy: `see <https://2.python-requests.org/en/latest/user/advanced/#proxies>`__.

169+ - SOCKS proxies are also supported: `see <https://2.python-requests.org/en/latest/user/advanced/#socks>`__

170+

169171``source_ips`` :

170172 If you use multiple network interfaces, define from which IP the requests must

171− be made. Example:

172−

173− * ``0.0.0.0`` any local IPv4 address.

174− * ``::`` any local IPv6 address.

175− * ``192.168.0.1``

176− * ``[ 192.168.0.1, 192.168.0.2 ]`` these two specific IP addresses

177− * ``fe80::60a2:1691:e5a2:ee1f``

178− * ``fe80::60a2:1691:e5a2:ee1f/126`` all IP addresses in this network.

179− * ``[ 192.168.0.1, fe80::/126 ]``

180−

181−``retries`` :

182− Number of retry in case of an HTTP error.

183− On each retry, searx uses an different proxy and source ip.

184−

185−``retry_on_http_error`` :

186− Retry request on some HTTP status code.

187−

188− Example:

189−

190− * ``true`` : on HTTP status code between 400 and 599.

191− * ``403`` : on HTTP status code 403.

192− * ``[403, 429]``: on HTTP status code 403 and 429.

193−

194−``enable_http2`` :

195− Enable by default. Set to ``False`` to disable HTTP/2.

196−

197−``max_redirects`` :

198− 30 by default. Maximum redirect before it is an error.

173+ be made. This parameter is ignored when ``proxies`` is set.

199174

200175

201176``locales:``

@@ -241,13 +216,6 @@ Engine settings

241216 api_key : 'apikey'

242217 disabled : True

243218 language : en_US

244− #enable_http: False

245− #enable_http2: False

246− #retries: 1

247− #retry_on_http_error: True # or 403 or [404, 429]

248− #max_connections: 100

249− #max_keepalive_connections: 10

250− #keepalive_expiry: 5.0

251219 #proxies:

252220 # http:

253221 # - http://proxy1:8080

@@ -302,12 +270,6 @@ Engine settings

302270``display_error_messages`` : default ``True``

303271 When an engine returns an error, the message is displayed on the user interface.

304272

305−``network``: optional

306− Use the network configuration from another engine.

307− In addition, there are two default networks:

308− * ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses)

309− * ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses)

310−

311273.. note::

312274

313275 A few more options are possible, but they are pretty specific to some

docs/admin/update-searx.rst+2−2

@@ -23,7 +23,7 @@ How to update depends on the :ref:`installation` method. If you have used the

2323

2424.. code:: bash

2525

26− sudo -H ./utils/morty.sh update morty

26+ $ sudo -H ./utils/morty.sh update morty

2727

2828.. _inspect searx:

2929

@@ -55,5 +55,5 @@ How to debug depends on the :ref:`installation` method. If you have used the

5555

5656.. code:: bash

5757

58− sudo -H ./utils/morty.sh inspect service

58+ $ sudo -H ./utils/morty.sh inspect service

5959

docs/blog/index.rst+3−5

@@ -3,15 +3,13 @@ Blog

33====

55.. toctree::

6− :titlesonly:

7− :reversed:

6+ :maxdepth: 2

7+ :caption: Contents

9+ lxcdev-202006

910 python3

1011 admin

1112 intro-offline

1213 private-engines

13− lxcdev-202006

1414 command-line-engines

1515 search-indexer-engines

16− sql-engines

17− search-database-engines

docs/blog/search-database-engines.rstdeleted−95

@@ -1,95 +0,0 @@

1−===============================

2−Query more of your NoSQL stores

3−===============================

4−

5−From now on, searx lets you to query your NoSQL data stores:

6−

7−* `Redis`_

8−* `MongoDB`_

9−

10−The reference configuration of the engines are included ``settings.yml`` just commented out,

11−as you have to set various options and install dependencies before using them.

12−

13−By default, the engines use ``key-value`` template for displaying results.

14−If you are not satisfied with the original result layout,

15−you can use your owm template by placing the template under

16−``searx/templates/{theme_name}/result_templates/{template_name}`` and setting

17−``result_template`` attribute to ``{template_name}``.

18−

19−Futhermore, if you do not want to expose these engines on a public instance, you can

20−still add them and limit the access by setting ``tokens`` as described in the `blog post about

21−private engines`_.

22−

23−Configuring searx to use the stores

24−===================================

25−

26−NoSQL data stores are used for storing arbitrary data without first defining their

27−structure.

28−

29−Redis

30−-----

31−

32−Reqired package: ``redis``

33−

34−Redis is a key value based data store usually stored in memory.

35−

36−Select a database to search in and set its index in the option ``db``. You can

37−either look for exact matches or use partial keywords to find what you are looking for

38−by configuring ``exact_match_only``.

39−

40−In this example you can search for exact matches in your first database:

41−

42−.. code:: yaml

43−

44− - name : mystore

45− engine : redis_server

46− exact_match_only : True

47− host : 127.0.0.1

48− port : 6379

49− password : secret-password

50− db : 0

51− shortcut : rds

52− enable_http : True

53−

54−

55−MongoDB

56−-------

57−

58−Required package: ``pymongo``

59−

60−MongoDB is a document based database program that handles JSON like data.

61−

62−In order to query MongoDB, you have to select a ``database`` and a ``collection``. Furthermore,

63−you have to select a ``key`` that is going to be searched. MongoDB also supports the option ``exact_match_only``, so configure it

64−as you wish.

65−

66−Above is an example configuration for using a MongoDB collection:

67−

68−.. code:: yaml

69−

70− - name : mymongo

71− engine : mongodb

72− shortcut : md

73− host : '127.0.0.1'

74− port : 27017

75− database : personal

76− collection : income

77− key : month

78− enable_http: True

79−

80−

81−Acknowledgement

82−===============

83−

84−This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .

85−

86−.. _Redis: https://redis.io/

87−.. _MongoDB: https://mongodb.com/

88−.. _blog post about private engines: private-engines.html#private-engines

89−.. _Search and Discovery Fund: https://nlnet.nl/discovery

90−.. _NLnet Foundation: https://nlnet.nl/

91−

92−

93−| Happy hacking.

94−| kvch // 2021.07.13 23:16

95−

docs/blog/sql-engines.rstdeleted−117

@@ -1,117 +0,0 @@

1−=================

2−Query SQL servers

3−=================

4−

5−Now you can query SQL servers using searx. The following ones are supported:

6−

7−* `PostgreSQL`_

8−* `MySQL`_

9−* `SQLite`_

10−

11−All of the engines above are added to ``settings.yml`` just commented out, as you have to

12−set the required attributes for the engines, e.g. ``database``. By default, the engines use

13−``key-value`` template for displaying results. If you are not satisfied with the original result layout,

14−you can use your owm template by placing the template under

15−``searx/templates/{theme_name}/result_templates/{template_name}`` and setting

16−``result_template`` attribute to ``{template_name}``.

17−

18−As mentioned in previous blog posts, if you do not wish to expose these engines on a

19−public instance, you can still add them and limit the access by setting ``tokens``

20−as described in the `blog post about private engines`_.

21−

22−Configure the engines

23−=====================

24−

25−The configuration of the new database engines are similar. You must put a valid

26−SELECT SQL query in ``query_str``. At the moment you can only bind at most

27−one parameter in your query. By setting the attribute ``limit`` you can

28−define how many results you want from the SQL server. Basically, it

29−is the same as the LIMIT keyword in SQL.

30−

31−Please, do not include LIMIT or OFFSET in your SQL query as the engines

32−rely on these keywords during paging. If you want to configure the number of returned results

33−use the option ``limit``.

34−

35−PostgreSQL

36−----------

37−

38−PostgreSQL is a powerful and robust open source database.

39−

40−Before configuring the PostgreSQL engine, you must install the dependency ``psychopg2``.

41−

42−You can find an example configuration below:

43−

44−.. code:: yaml

45−

46− - name : postgresql

47− engine : postgresql

48− database : my_database

49− username : searx

50− password : password

51− query_str : 'SELECT * from my_table WHERE my_column = %(query)s'

52− shortcut : psql

53−

54−

55−MySQL

56−-----

57−

58−MySQL is said to be the most popular open source database.

59−

60−Before enabling MySQL engine, you must install the package ``mysql-connector-python``.

61−

62−The authentication plugin is configurable by setting ``auth_plugin`` in the attributes.

63−By default it is set to ``caching_sha2_password``.

64−

65−This is an example configuration for quering a MySQL server:

66−

67−.. code:: yaml

68−

69− - name : mysql

70− engine : mysql_server

71− database : my_database

72− username : searx

73− password : password

74− limit : 5

75− query_str : 'SELECT * from my_table WHERE my_column=%(query)s'

76− shortcut : mysql

77−

78−

79−SQLite

80−------

81−

82−SQLite is a small, fast and reliable SQL database engine. It does not require

83−any extra dependency.

84−

85−You can read from your database ``my_database`` using this example configuration:

86−

87−.. code:: yaml

88−

89− - name : sqlite

90− engine : sqlite

91− shortcut: sq

92− database : my_database

93− query_str : 'SELECT * FROM my_table WHERE my_column=:query'

94−

95−

96−Next steps

97−==========

98−

99−The next step is to add support for more data stores, e.g. Redis and MongoDB.

100−

101−Acknowledgement

102−===============

103−

104−This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .

105−

106−.. _PostgreSQL: https://www.postgresql.org/

107−.. _MySQL: https://www.mysql.com/

108−.. _SQLite: https://www.sqlite.org/index.html

109−.. _blog post about private engines: private-engines.html#private-engines

110−.. _Search and Discovery Fund: https://nlnet.nl/discovery

111−.. _NLnet Foundation: https://nlnet.nl/

112−

113−

114−| Happy hacking.

115−| kvch // 2021.05.23 23:16

116−

117−

docs/conf.py+1−2

@@ -10,7 +10,7 @@ from searx.version import VERSION_STRING

1010# Project --------------------------------------------------------------

1111

1212project = u'searx'

13−copyright = u'2015-2021, Adam Tauber, Noémi Ványi'

13+copyright = u'2015-2020, Adam Tauber, Noémi Ványi'

1414author = u'Adam Tauber'

1515release, version = VERSION_STRING, VERSION_STRING

1616highlight_language = 'none'

@@ -101,7 +101,6 @@ imgmath_font_size = 14

101101

102102html_theme_options = {"index_sidebar_logo": True}

103103html_context = {"project_links": [] }

104−html_context["project_links"].append(ProjectLink("Blog", "blog/index.html"))

105104if brand.GIT_URL:

106105 html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))

107106if brand.WIKI_URL:

docs/dev/engine_overview.rst+1−1

@@ -171,7 +171,7 @@ headers set HTTP header information

171171data set HTTP data information

172172cookies set HTTP cookies

173173verify bool Performing SSL-Validity check

174−follow_redirects bool Follow redirects

174+allow_redirects bool Follow redirects

175175max_redirects int maximum redirects, hard limit

176176soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

177177raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300

manage+5−6

@@ -13,7 +13,7 @@ source_dot_config

1313

1414PY_SETUP_EXTRAS='[test]'

1515NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"

16−GECKODRIVER_VERSION="v0.30.0"

16+GECKODRIVER_VERSION="v0.28.0"

1717# SPHINXOPTS=

1818

1919# These py files are linted by test.pylint(), all other files are linted by

@@ -32,7 +32,6 @@ PYLINT_FILES=(

3232 searx/engines/meilisearch.py

3333 searx/engines/solidtorrents.py

3434 searx/engines/solr.py

35− searx/engines/sqlite.py

3635 searx/engines/springer.py

3736 searx/engines/google_scholar.py

3837 searx/engines/yahoo_news.py

@@ -224,7 +223,7 @@ docker.build() {

224223

225224 # build Docker image

226225 build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

227− docker build \

226+ sudo docker build \

228227 --build-arg GIT_URL="${GIT_URL}" \

229228 --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

230229 --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

@@ -236,8 +235,8 @@ docker.build() {

236235 -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

237236

238237 if [ "$1" = "push" ]; then

239− docker push "${SEARX_IMAGE_NAME}:latest"

240− docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

238+ sudo docker push "${SEARX_IMAGE_NAME}:latest"

239+ sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

241240 fi

242241 )

243242 dump_return $?

@@ -400,7 +399,7 @@ test.pylint() {

400399 pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

401400 --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

402401 --ignore=searx/engines \

403− tests

402+ searx tests

404403 )

405404 dump_return $?

406405}

requirements-dev.txt+12−14

@@ -1,20 +1,18 @@

11mock==4.0.3

22nose2[coverage_plugin]==0.10.0

33cov-core==1.15.0

4−pycodestyle==2.8.0

5−pylint==2.12.2

6−splinter==0.17.0

7−transifex-client==0.14.3; python_version < '3.10'

8−transifex-client==0.12.1; python_version == '3.10'

9−selenium==4.1.0

10−twine==3.7.1

11−Pallets-Sphinx-Themes==2.0.2

4+pycodestyle==2.7.0

5+pylint==2.7.4

6+splinter==0.14.0

7+transifex-client==0.14.2

8+selenium==3.141.0

9+twine==3.4.1

10+Pallets-Sphinx-Themes==1.2.3

1211docutils==0.16

13−Sphinx==4.4.0

14−sphinx-issues==3.0.1

15−sphinx-jinja==1.4.0

16−sphinx-tabs==3.2.0

12+Sphinx==3.5.4

13+sphinx-issues==1.2.0

14+sphinx-jinja==1.1.1

15+sphinx-tabs==2.1.0

1716sphinxcontrib-programoutput==0.17

1817sphinx-autobuild==2021.3.14

19−linuxdoc==20211220

20−aiounittest==1.4.1

18+linuxdoc==20210324

requirements.txt+10−14

@@ -1,16 +1,12 @@

1−certifi==2021.10.8

2−babel==2.9.1

1+certifi==2020.12.05

2+babel==2.9.0

33flask-babel==2.0.0

4−flask==2.0.2

5−jinja2==3.0.3

6−lxml==4.7.1

4+flask==1.1.2

5+idna==2.10

6+jinja2==2.11.3

7+lxml==4.6.3

78pygments==2.8.0

8−python-dateutil==2.8.2

9−pyyaml==6.0

10−httpx[http2]==0.21.3

11−Brotli==1.0.9

12−uvloop==0.16.0; python_version >= '3.7'

13−uvloop==0.14.0; python_version < '3.7'

14−httpx-socks[asyncio]==0.7.2

15−langdetect==1.0.9

16−setproctitle==1.2.2

9+python-dateutil==2.8.1

10+pyyaml==5.4.1

11+requests[socks]==2.25.1

12+langdetect==1.0.8

searx/autocomplete.py+10−28

@@ -20,12 +20,10 @@ from lxml import etree

2020from json import loads

2121from urllib.parse import urlencode

2222

23−from httpx import HTTPError

24−

23+from requests import RequestException

2524

2625from searx import settings

27−from searx.data import ENGINES_LANGUAGES

28−from searx.network import get as http_get

26+from searx.poolrequests import get as http_get

2927from searx.exceptions import SearxEngineResponseException

3028

3129

@@ -36,22 +34,6 @@ def get(*args, **kwargs):

3634 return http_get(*args, **kwargs)

3735

3836

39−def brave(query, lang):

40− # brave search autocompleter

41− url = 'https://search.brave.com/api/suggest?{query}'

42−

43− resp = get(url.format(query=urlencode({'q': query})))

44−

45− results = []

46−

47− if resp.ok:

48− data = loads(resp.text)

49− for suggestion in data[1]:

50− results.append(suggestion)

51−

52− return results

53−

54−

5537def dbpedia(query, lang):

5638 # dbpedia autocompleter, no HTTPS

5739 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

@@ -94,11 +76,12 @@ def google(query, lang):

9476

9577def startpage(query, lang):

9678 # startpage autocompleter

97− lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')

98− url = 'https://startpage.com/suggestions?{query}'

99− resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))

100− data = resp.json()

101− return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

79+ url = 'https://startpage.com/do/suggest?{query}'

80+

81+ resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')

82+ if len(resp) > 1:

83+ return resp

84+ return []

10285

10386

10487def swisscows(query, lang):

@@ -136,8 +119,7 @@ def wikipedia(query, lang):

136119 return []

137120

138121

139−backends = {'brave': brave,

140− 'dbpedia': dbpedia,

122+backends = {'dbpedia': dbpedia,

141123 'duckduckgo': duckduckgo,

142124 'google': google,

143125 'startpage': startpage,

@@ -154,5 +136,5 @@ def search_autocomplete(backend_name, query, lang):

154136

155137 try:

156138 return backend(query, lang)

157− except (HTTPError, SearxEngineResponseException):

139+ except (RequestException, SearxEngineResponseException):

158140 return []

searx/data/__init__.py+14−39

@@ -1,50 +1,25 @@

1−# SPDX-License-Identifier: AGPL-3.0-or-later

2−# lint: pylint

3−"""This module holds the *data* created by::

4−

5− make data.all

6−

7−"""

8−

9−__all__ = [

10− 'ENGINES_LANGUAGES',

11− 'CURRENCIES',

12− 'USER_AGENTS',

13− 'EXTERNAL_URLS',

14− 'WIKIDATA_UNITS',

15− 'EXTERNAL_BANGS',

16− 'OSM_KEYS_TAGS',

17− 'ahmia_blacklist_loader',

18−]

19−

201import json

212from pathlib import Path

223

5+__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'EXTERNAL_URLS', 'WIKIDATA_UNITS', 'EXTERNAL_BANGS',

6+ 'bangs_loader', 'ahmia_blacklist_loader']

237data_dir = Path(__file__).parent

248

259

26−def _load(filename):

27− with open(data_dir / filename, encoding='utf-8') as f:

28− return json.load(f)

10+def load(filename):

11+ with open(data_dir / filename, encoding='utf-8') as fd:

12+ return json.load(fd)

2913

3014

3115def ahmia_blacklist_loader():

32− """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion

33− names. The MD5 values are fetched by::

34−

35− searx_extra/update/update_ahmia_blacklist.py

36−

37− This function is used by :py:mod:`searx.plugins.ahmia_filter`.

38−

39− """

40− with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as f:

41− return f.read().split()

16+ with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:

17+ return fd.read().split()

4218

4319

44−ENGINES_LANGUAGES = _load('engines_languages.json')

45−CURRENCIES = _load('currencies.json')

46−USER_AGENTS = _load('useragents.json')

47−EXTERNAL_URLS = _load('external_urls.json')

48−WIKIDATA_UNITS = _load('wikidata_units.json')

49−EXTERNAL_BANGS = _load('external_bangs.json')

50−OSM_KEYS_TAGS = _load('osm_keys_tags.json')

20+ENGINES_LANGUAGES = load('engines_languages.json')

21+CURRENCIES = load('currencies.json')

22+USER_AGENTS = load('useragents.json')

23+EXTERNAL_URLS = load('external_urls.json')

24+WIKIDATA_UNITS = load('wikidata_units.json')

25+EXTERNAL_BANGS = load('external_bangs.json')

searx/data/ahmia_blacklist.txt−11140