master..426ce34 · searx

.github/workflows/data-update.yml+1−1

since splitsnapshots

unifiedsplit

@@ -29,7 +29,7 @@ jobs:

29 - name: Set up Python

30 uses: actions/setup-python@v2

31 with:

32− python-version: '3.10'

32+ python-version: '3.9'

33 architecture: 'x64'

34

35 - name: Install Python dependencies

.github/workflows/integration.yml+3−32

@@ -13,7 +13,7 @@ jobs:

13 strategy:

14 matrix:

15 os: [ubuntu-20.04]

16− python-version: [3.7, 3.8, 3.9, "3.10"]

16+ python-version: [3.6, 3.7, 3.8, 3.9]

17 steps:

18 - name: Checkout

19 uses: actions/checkout@v2

@@ -56,17 +56,6 @@ jobs:

56 uses: actions/checkout@v2

57 - name: Install Ubuntu packages

58 run: sudo ./utils/searx.sh install packages

59− - name: Set up Python

60− uses: actions/setup-python@v2

61− with:

62− python-version: '3.9'

63− architecture: 'x64'

64− - name: Cache Python dependencies

65− id: cache-python

66− uses: actions/cache@v2

67− with:

68− path: ./local

69− key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

70 - name: Install node dependencies

59 - name: Install node dependencies

71 run: make V=1 node.env

60 run: make V=1 node.env

72 - name: Build themes

61 - name: Build themes

@@ -79,21 +68,14 @@ jobs:

79 - name: Checkout

68 - name: Checkout

80 uses: actions/checkout@v2

69 uses: actions/checkout@v2

81 with:

70 with:

82− fetch-depth: '0'

83 persist-credentials: false

71 persist-credentials: false

84 - name: Install Ubuntu packages

72 - name: Install Ubuntu packages

85 run: sudo ./utils/searx.sh install buildhost

73 run: sudo ./utils/searx.sh install buildhost

86 - name: Set up Python

74 - name: Set up Python

87 uses: actions/setup-python@v2

75 uses: actions/setup-python@v2

88 with:

76 with:

89− python-version: '3.10'

77+ python-version: '3.9'

90 architecture: 'x64'

78 architecture: 'x64'

91− - name: Cache Python dependencies

92− id: cache-python

93− uses: actions/cache@v2

94− with:

95− path: ./local

96− key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

97 - name: Build documentation

79 - name: Build documentation

98 run: |

80 run: |

99 make V=1 docs.clean docs.html

81 make V=1 docs.clean docs.html

@@ -115,7 +97,7 @@ jobs:

115 - documentation

97 - documentation

116 env:

98 env:

117 DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

99 DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

118− runs-on: ubuntu-20.04

100+ runs-on: ubuntu-18.04

119 steps:

101 steps:

120 - name: Checkout

102 - name: Checkout

121 if: env.DOCKERHUB_USERNAME != null

103 if: env.DOCKERHUB_USERNAME != null

@@ -123,17 +105,6 @@ jobs:

123 with:

105 with:

124 # make sure "make docker.push" can get the git history

106 # make sure "make docker.push" can get the git history

125 fetch-depth: '0'

107 fetch-depth: '0'

126− - name: Set up Python

127− uses: actions/setup-python@v2

128− with:

129− python-version: '3.9'

130− architecture: 'x64'

131− - name: Cache Python dependencies

132− id: cache-python

133− uses: actions/cache@v2

134− with:

135− path: ./local

136− key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

137 - name: Set up QEMU

108 - name: Set up QEMU

138 if: env.DOCKERHUB_USERNAME != null

109 if: env.DOCKERHUB_USERNAME != null

139 uses: docker/setup-qemu-action@v1

110 uses: docker/setup-qemu-action@v1

.gitignore−2

@@ -26,5 +26,3 @@ dist/

26local/

27gh-pages/

28searx.egg-info/

29−.env

30−geckodriver.log

.pylintrc+1−1

@@ -59,7 +59,7 @@ confidence=

59# --enable=similarities". If you want to run only the classes checker, but have

60# no Warning level messages displayed, use"--disable=all --enable=classes

61# --disable=W"

62−disable=bad-whitespace, duplicate-code, consider-using-f-string

62+disable=bad-whitespace, duplicate-code

63

64# Enable the message, report, category or checker with the given id(s). You can

65# either give multiple identifier separated by comma (,) or put this option

AUTHORS.rst−1

@@ -165,4 +165,3 @@ generally made searx better:

165- @jhigginbotham

166- @xenrox

167- @OliveiraHermogenes

168−- Paul Alcock @Guilvareux

Dockerfile+4−5

@@ -1,4 +1,4 @@

1−FROM alpine:3.15

1+FROM alpine:3.12

2ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

3EXPOSE 8080

4VOLUME /etc/searx

@@ -46,19 +46,18 @@ RUN apk upgrade --no-cache \

46 uwsgi \

47 uwsgi-python3 \

48 brotli \

49− && pip3 install --upgrade pip wheel setuptools \

49+ && pip3 install --upgrade pip \

50 && pip3 install --no-cache -r requirements.txt \

51 && apk del build-dependencies \

52 && rm -rf /root/.cache

53

54−COPY searx ./searx

54+COPY --chown=searx:searx . .

55−COPY dockerfiles ./dockerfiles

56

55

57ARG TIMESTAMP_SETTINGS=0

56ARG TIMESTAMP_SETTINGS=0

58ARG TIMESTAMP_UWSGI=0

57ARG TIMESTAMP_UWSGI=0

59ARG VERSION_GITCOMMIT=unknown

58ARG VERSION_GITCOMMIT=unknown

60

59

61−RUN /usr/bin/python3 -m compileall -q searx; \

60+RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

62 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

61 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

63 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

62 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

64 if [ ! -z $VERSION_GITCOMMIT ]; then\

63 if [ ! -z $VERSION_GITCOMMIT ]; then\

README.rst+2−2

@@ -17,7 +17,7 @@

17|OpenCollective searx backers|

18|OpenCollective searx sponsors|

19

20−Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **sɜːks**.

20+Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **səːks**.

21

22.. _metasearch engine: https://en.wikipedia.org/wiki/Metasearch_engine

23

@@ -61,7 +61,7 @@ our homepage_.

61.. _homepage: https://searx.github.io/searx

62

63contact:

64− openhub_ // twitter_ // IRC: #searx @ Libera (irc.libera.chat)

64+ openhub_ // twitter_ // IRC: #searx @ freenode

65

66.. _openhub: https://www.openhub.net/p/searx

67.. _twitter: https://twitter.com/Searx_engine

docs/admin/settings.rst+13−51

@@ -130,12 +130,14 @@ Global Settings

130 request_timeout : 2.0 # default timeout in seconds, can be override by engine

131 # max_request_timeout: 10.0 # the maximum timeout in seconds

132 useragent_suffix : "" # informations like an email address to the administrator

133− pool_connections : 100 # Maximum number of allowable connections, or None for no limits. The default is 100.

133+ pool_connections : 100 # Number of different hosts

134− pool_maxsize : 10 # Number of allowable keep-alive connections, or None to always allow. The default is 10.

134+ pool_maxsize : 10 # Number of simultaneous requests by host

135− enable_http2: True # See https://www.python-httpx.org/http2/

136 # uncomment below section if you want to use a proxy

135 # uncomment below section if you want to use a proxy

137 # proxies:

136 # proxies:

138− # all://:

137+ # http:

138+ # - http://proxy1:8080

139+ # - http://proxy2:8080

140+ # https:

139 # - http://proxy1:8080

141 # - http://proxy1:8080

140 # - http://proxy2:8080

142 # - http://proxy2:8080

141 # uncomment below section only if you have more than one network interface

143 # uncomment below section only if you have more than one network interface

@@ -143,7 +145,6 @@ Global Settings

143 # source_ips:

145 # source_ips:

144 # - 1.1.1.1

146 # - 1.1.1.1

145 # - 1.1.1.2

147 # - 1.1.1.2

146− # - fe80::/126

147

148

149

149``request_timeout`` :

150``request_timeout`` :

@@ -156,46 +157,20 @@ Global Settings

156 Suffix to the user-agent searx uses to send requests to others engines. If an

157 Suffix to the user-agent searx uses to send requests to others engines. If an

157 engine wish to block you, a contact info here may be useful to avoid that.

158 engine wish to block you, a contact info here may be useful to avoid that.

158

159

159−``keepalive_expiry``:

160+.. _requests proxies: https://requests.readthedocs.io/en/latest/user/advanced/#proxies

160− Number of seconds to keep a connection in the pool. By default 5.0 seconds.

161+.. _PySocks: https://pypi.org/project/PySocks/

161−

162−.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying

163

162

164``proxies`` :

163``proxies`` :

165− Define one or more proxies you wish to use, see `httpx proxies`_.

164+ Define one or more proxies you wish to use, see `requests proxies`_.

166 If there are more than one proxy for one protocol (http, https),

165 If there are more than one proxy for one protocol (http, https),

167 requests to the engines are distributed in a round-robin fashion.

166 requests to the engines are distributed in a round-robin fashion.

168

167

168+ - Proxy: `see <https://2.python-requests.org/en/latest/user/advanced/#proxies>`__.

169+ - SOCKS proxies are also supported: `see <https://2.python-requests.org/en/latest/user/advanced/#socks>`__

170+

169``source_ips`` :

171``source_ips`` :

170 If you use multiple network interfaces, define from which IP the requests must

172 If you use multiple network interfaces, define from which IP the requests must

171− be made. Example:

173+ be made. This parameter is ignored when ``proxies`` is set.

172−

173− * ``0.0.0.0`` any local IPv4 address.

174− * ``::`` any local IPv6 address.

175− * ``192.168.0.1``

176− * ``[ 192.168.0.1, 192.168.0.2 ]`` these two specific IP addresses

177− * ``fe80::60a2:1691:e5a2:ee1f``

178− * ``fe80::60a2:1691:e5a2:ee1f/126`` all IP addresses in this network.

179− * ``[ 192.168.0.1, fe80::/126 ]``

180−

181−``retries`` :

182− Number of retry in case of an HTTP error.

183− On each retry, searx uses an different proxy and source ip.

184−

185−``retry_on_http_error`` :

186− Retry request on some HTTP status code.

187−

188− Example:

189−

190− * ``true`` : on HTTP status code between 400 and 599.

191− * ``403`` : on HTTP status code 403.

192− * ``[403, 429]``: on HTTP status code 403 and 429.

193−

194−``enable_http2`` :

195− Enable by default. Set to ``False`` to disable HTTP/2.

196−

197−``max_redirects`` :

198− 30 by default. Maximum redirect before it is an error.

199

174

200

175

201``locales:``

176``locales:``

@@ -241,13 +216,6 @@ Engine settings

241 api_key : 'apikey'

216 api_key : 'apikey'

242 disabled : True

217 disabled : True

243 language : en_US

218 language : en_US

244− #enable_http: False

245− #enable_http2: False

246− #retries: 1

247− #retry_on_http_error: True # or 403 or [404, 429]

248− #max_connections: 100

249− #max_keepalive_connections: 10

250− #keepalive_expiry: 5.0

251 #proxies:

219 #proxies:

252 # http:

220 # http:

253 # - http://proxy1:8080

221 # - http://proxy1:8080

@@ -302,12 +270,6 @@ Engine settings

302``display_error_messages`` : default ``True``

270``display_error_messages`` : default ``True``

303 When an engine returns an error, the message is displayed on the user interface.

271 When an engine returns an error, the message is displayed on the user interface.

304

272

305−``network``: optional

306− Use the network configuration from another engine.

307− In addition, there are two default networks:

308− * ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses)

309− * ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses)

310−

311.. note::

273.. note::

312

274

313 A few more options are possible, but they are pretty specific to some

275 A few more options are possible, but they are pretty specific to some

docs/admin/update-searx.rst+2−2

@@ -23,7 +23,7 @@ How to update depends on the :ref:`installation` method. If you have used the

23

24.. code:: bash

25

26− sudo -H ./utils/morty.sh update morty

26+ $ sudo -H ./utils/morty.sh update morty

27

28.. _inspect searx:

29

@@ -55,5 +55,5 @@ How to debug depends on the :ref:`installation` method. If you have used the

55

56.. code:: bash

57

58− sudo -H ./utils/morty.sh inspect service

58+ $ sudo -H ./utils/morty.sh inspect service

59

docs/blog/index.rst+3−5

@@ -3,15 +3,13 @@ Blog

3====

4

5.. toctree::

6− :titlesonly:

6+ :maxdepth: 2

7− :reversed:

7+ :caption: Contents

8

9+ lxcdev-202006

9 python3

10 python3

10 admin

11 admin

11 intro-offline

12 intro-offline

12 private-engines

13 private-engines

13− lxcdev-202006

14 command-line-engines

15 search-indexer-engines

16− sql-engines

17− search-database-engines

docs/blog/search-database-engines.rstdeleted−95

@@ -1,95 +0,0 @@

1−===============================

2−Query more of your NoSQL stores

3−===============================

4−

5−From now on, searx lets you to query your NoSQL data stores:

6−

7−* `Redis`_

8−* `MongoDB`_

9−

10−The reference configuration of the engines are included ``settings.yml`` just commented out,

11−as you have to set various options and install dependencies before using them.

12−

13−By default, the engines use ``key-value`` template for displaying results.

14−If you are not satisfied with the original result layout,

15−you can use your owm template by placing the template under

16−``searx/templates/{theme_name}/result_templates/{template_name}`` and setting

17−``result_template`` attribute to ``{template_name}``.

18−

19−Futhermore, if you do not want to expose these engines on a public instance, you can

20−still add them and limit the access by setting ``tokens`` as described in the `blog post about

21−private engines`_.

22−

23−Configuring searx to use the stores

24−===================================

25−

26−NoSQL data stores are used for storing arbitrary data without first defining their

27−structure.

28−

29−Redis

30−-----

31−

32−Reqired package: ``redis``

33−

34−Redis is a key value based data store usually stored in memory.

35−

36−Select a database to search in and set its index in the option ``db``. You can

37−either look for exact matches or use partial keywords to find what you are looking for

38−by configuring ``exact_match_only``.

39−

40−In this example you can search for exact matches in your first database:

41−

42−.. code:: yaml

43−

44− - name : mystore

45− engine : redis_server

46− exact_match_only : True

47− host : 127.0.0.1

48− port : 6379

49− password : secret-password

50− db : 0

51− shortcut : rds

52− enable_http : True

53−

54−

55−MongoDB

56−-------

57−

58−Required package: ``pymongo``

59−

60−MongoDB is a document based database program that handles JSON like data.

61−

62−In order to query MongoDB, you have to select a ``database`` and a ``collection``. Furthermore,

63−you have to select a ``key`` that is going to be searched. MongoDB also supports the option ``exact_match_only``, so configure it

64−as you wish.

65−

66−Above is an example configuration for using a MongoDB collection:

67−

68−.. code:: yaml

69−

70− - name : mymongo

71− engine : mongodb

72− shortcut : md

73− host : '127.0.0.1'

74− port : 27017

75− database : personal

76− collection : income

77− key : month

78− enable_http: True

79−

80−

81−Acknowledgement

82−===============

83−

84−This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .

85−

86−.. _Redis: https://redis.io/

87−.. _MongoDB: https://mongodb.com/

88−.. _blog post about private engines: private-engines.html#private-engines

89−.. _Search and Discovery Fund: https://nlnet.nl/discovery

90−.. _NLnet Foundation: https://nlnet.nl/

91−

92−

93−| Happy hacking.

94−| kvch // 2021.07.13 23:16

95−

docs/blog/sql-engines.rstdeleted−117

@@ -1,117 +0,0 @@

1−=================

2−Query SQL servers

3−=================

4−

5−Now you can query SQL servers using searx. The following ones are supported:

6−

7−* `PostgreSQL`_

8−* `MySQL`_

9−* `SQLite`_

10−

11−All of the engines above are added to ``settings.yml`` just commented out, as you have to

12−set the required attributes for the engines, e.g. ``database``. By default, the engines use

13−``key-value`` template for displaying results. If you are not satisfied with the original result layout,

14−you can use your owm template by placing the template under

15−``searx/templates/{theme_name}/result_templates/{template_name}`` and setting

16−``result_template`` attribute to ``{template_name}``.

17−

18−As mentioned in previous blog posts, if you do not wish to expose these engines on a

19−public instance, you can still add them and limit the access by setting ``tokens``

20−as described in the `blog post about private engines`_.

21−

22−Configure the engines

23−=====================

24−

25−The configuration of the new database engines are similar. You must put a valid

26−SELECT SQL query in ``query_str``. At the moment you can only bind at most

27−one parameter in your query. By setting the attribute ``limit`` you can

28−define how many results you want from the SQL server. Basically, it

29−is the same as the LIMIT keyword in SQL.

30−

31−Please, do not include LIMIT or OFFSET in your SQL query as the engines

32−rely on these keywords during paging. If you want to configure the number of returned results

33−use the option ``limit``.

34−

35−PostgreSQL

36−----------

37−

38−PostgreSQL is a powerful and robust open source database.

39−

40−Before configuring the PostgreSQL engine, you must install the dependency ``psychopg2``.

41−

42−You can find an example configuration below:

43−

44−.. code:: yaml

45−

46− - name : postgresql

47− engine : postgresql

48− database : my_database

49− username : searx

50− password : password

51− query_str : 'SELECT * from my_table WHERE my_column = %(query)s'

52− shortcut : psql

53−

54−

55−MySQL

56−-----

57−

58−MySQL is said to be the most popular open source database.

59−

60−Before enabling MySQL engine, you must install the package ``mysql-connector-python``.

61−

62−The authentication plugin is configurable by setting ``auth_plugin`` in the attributes.

63−By default it is set to ``caching_sha2_password``.

64−

65−This is an example configuration for quering a MySQL server:

66−

67−.. code:: yaml

68−

69− - name : mysql

70− engine : mysql_server

71− database : my_database

72− username : searx

73− password : password

74− limit : 5

75− query_str : 'SELECT * from my_table WHERE my_column=%(query)s'

76− shortcut : mysql

77−

78−

79−SQLite

80−------

81−

82−SQLite is a small, fast and reliable SQL database engine. It does not require

83−any extra dependency.

84−

85−You can read from your database ``my_database`` using this example configuration:

86−

87−.. code:: yaml

88−

89− - name : sqlite

90− engine : sqlite

91− shortcut: sq

92− database : my_database

93− query_str : 'SELECT * FROM my_table WHERE my_column=:query'

94−

95−

96−Next steps

97−==========

98−

99−The next step is to add support for more data stores, e.g. Redis and MongoDB.

100−

101−Acknowledgement

102−===============

103−

104−This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .

105−

106−.. _PostgreSQL: https://www.postgresql.org/

107−.. _MySQL: https://www.mysql.com/

108−.. _SQLite: https://www.sqlite.org/index.html

109−.. _blog post about private engines: private-engines.html#private-engines

110−.. _Search and Discovery Fund: https://nlnet.nl/discovery

111−.. _NLnet Foundation: https://nlnet.nl/

112−

113−

114−| Happy hacking.

115−| kvch // 2021.05.23 23:16

116−

117−

docs/conf.py+1−2

@@ -10,7 +10,7 @@ from searx.version import VERSION_STRING

10# Project --------------------------------------------------------------

11

12project = u'searx'

13−copyright = u'2015-2021, Adam Tauber, Noémi Ványi'

13+copyright = u'2015-2020, Adam Tauber, Noémi Ványi'

14author = u'Adam Tauber'

15release, version = VERSION_STRING, VERSION_STRING

16highlight_language = 'none'

@@ -101,7 +101,6 @@ imgmath_font_size = 14

101

102html_theme_options = {"index_sidebar_logo": True}

103html_context = {"project_links": [] }

104−html_context["project_links"].append(ProjectLink("Blog", "blog/index.html"))

105if brand.GIT_URL:

104if brand.GIT_URL:

106 html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))

105 html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))

107if brand.WIKI_URL:

106if brand.WIKI_URL:

docs/dev/engine_overview.rst+1−1

@@ -171,7 +171,7 @@ headers set HTTP header information

171data set HTTP data information

172cookies set HTTP cookies

173verify bool Performing SSL-Validity check

174−follow_redirects bool Follow redirects

174+allow_redirects bool Follow redirects

175max_redirects int maximum redirects, hard limit

176soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

177raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300

manage+5−6

@@ -13,7 +13,7 @@ source_dot_config

13

14PY_SETUP_EXTRAS='[test]'

15NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"

16−GECKODRIVER_VERSION="v0.30.0"

16+GECKODRIVER_VERSION="v0.28.0"

17# SPHINXOPTS=

18

19# These py files are linted by test.pylint(), all other files are linted by

@@ -32,7 +32,6 @@ PYLINT_FILES=(

32 searx/engines/meilisearch.py

33 searx/engines/solidtorrents.py

34 searx/engines/solr.py

35− searx/engines/sqlite.py

36 searx/engines/springer.py

35 searx/engines/springer.py

37 searx/engines/google_scholar.py

36 searx/engines/google_scholar.py

38 searx/engines/yahoo_news.py

37 searx/engines/yahoo_news.py

@@ -224,7 +223,7 @@ docker.build() {

224

223

225 # build Docker image

224 # build Docker image

226 build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

225 build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

227− docker build \

226+ sudo docker build \

228 --build-arg GIT_URL="${GIT_URL}" \

227 --build-arg GIT_URL="${GIT_URL}" \

229 --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

228 --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

230 --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

229 --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

@@ -236,8 +235,8 @@ docker.build() {

236 -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

235 -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

237

236

238 if [ "$1" = "push" ]; then

237 if [ "$1" = "push" ]; then

239− docker push "${SEARX_IMAGE_NAME}:latest"

238+ sudo docker push "${SEARX_IMAGE_NAME}:latest"

240− docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

239+ sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

241 fi

240 fi

242 )

241 )

243 dump_return $?

242 dump_return $?

@@ -400,7 +399,7 @@ test.pylint() {

400 pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

399 pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

401 --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

400 --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

402 --ignore=searx/engines \

401 --ignore=searx/engines \

403− tests

402+ searx tests

404 )

403 )

405 dump_return $?

404 dump_return $?

406}

405}

requirements-dev.txt+12−14

@@ -1,20 +1,18 @@

1mock==4.0.3

2nose2[coverage_plugin]==0.10.0

3cov-core==1.15.0

4−pycodestyle==2.8.0

4+pycodestyle==2.7.0

5−pylint==2.12.2

5+pylint==2.7.4

6−splinter==0.17.0

6+splinter==0.14.0

7−transifex-client==0.14.3; python_version < '3.10'

7+transifex-client==0.14.2

8−transifex-client==0.12.1; python_version == '3.10'

9−selenium==4.1.0

8+selenium==3.141.0

10−twine==3.7.1

9+twine==3.4.1

11−Pallets-Sphinx-Themes==2.0.2

10+Pallets-Sphinx-Themes==1.2.3

12docutils==0.16

11docutils==0.16

13−Sphinx==4.4.0

12+Sphinx==3.5.4

14−sphinx-issues==3.0.1

13+sphinx-issues==1.2.0

15−sphinx-jinja==1.4.0

14+sphinx-jinja==1.1.1

16−sphinx-tabs==3.2.0

15+sphinx-tabs==2.1.0

17sphinxcontrib-programoutput==0.17

16sphinxcontrib-programoutput==0.17

18sphinx-autobuild==2021.3.14

17sphinx-autobuild==2021.3.14

19−linuxdoc==20211220

18+linuxdoc==20210324

20−aiounittest==1.4.1

requirements.txt+9−13

@@ -1,16 +1,12 @@

1−certifi==2021.10.8

1+certifi==2020.12.05

2babel==2.9.1

3flask-babel==2.0.0

4−flask==2.0.2

4+flask==1.1.2

5+idna==2.10

5−jinja2==3.0.3

6+jinja2==2.11.3

6−lxml==4.7.1

7+lxml==4.6.3

7pygments==2.8.0

8pygments==2.8.0

8−python-dateutil==2.8.2

9+python-dateutil==2.8.1

9−pyyaml==6.0

10+pyyaml==5.4.1

10−httpx[http2]==0.21.3

11+requests[socks]==2.25.1

11−Brotli==1.0.9

12−uvloop==0.16.0; python_version >= '3.7'

13−uvloop==0.14.0; python_version < '3.7'

14−httpx-socks[asyncio]==0.7.2

15−langdetect==1.0.9

12+langdetect==1.0.8

16−setproctitle==1.2.2

searx/autocomplete.py+10−28

@@ -20,12 +20,10 @@ from lxml import etree

20from json import loads

21from urllib.parse import urlencode

22

23−from httpx import HTTPError

23+from requests import RequestException

24−

25

24

26from searx import settings

25from searx import settings

27−from searx.data import ENGINES_LANGUAGES

28−from searx.network import get as http_get

26+from searx.poolrequests import get as http_get

29from searx.exceptions import SearxEngineResponseException

27from searx.exceptions import SearxEngineResponseException

30

28

31

29

@@ -36,22 +34,6 @@ def get(*args, **kwargs):

36 return http_get(*args, **kwargs)

34 return http_get(*args, **kwargs)

37

35

38

36

39−def brave(query, lang):

40− # brave search autocompleter

41− url = 'https://search.brave.com/api/suggest?{query}'

42−

43− resp = get(url.format(query=urlencode({'q': query})))

44−

45− results = []

46−

47− if resp.ok:

48− data = loads(resp.text)

49− for suggestion in data[1]:

50− results.append(suggestion)

51−

52− return results

53−

54−

55def dbpedia(query, lang):

37def dbpedia(query, lang):

56 # dbpedia autocompleter, no HTTPS

38 # dbpedia autocompleter, no HTTPS

57 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

39 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

@@ -94,11 +76,12 @@ def google(query, lang):

94

76

95def startpage(query, lang):

77def startpage(query, lang):

96 # startpage autocompleter

78 # startpage autocompleter

97− lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')

98− url = 'https://startpage.com/suggestions?{query}'

79+ url = 'https://startpage.com/do/suggest?{query}'

80+

99− resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))

81+ resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')

100− data = resp.json()

82+ if len(resp) > 1:

101− return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

83+ return resp

84+ return []

102

85

103

86

104def swisscows(query, lang):

87def swisscows(query, lang):

@@ -136,8 +119,7 @@ def wikipedia(query, lang):

136 return []

119 return []

137

120

138

121

139−backends = {'brave': brave,

140− 'dbpedia': dbpedia,

122+backends = {'dbpedia': dbpedia,

141 'duckduckgo': duckduckgo,

123 'duckduckgo': duckduckgo,

142 'google': google,

124 'google': google,

143 'startpage': startpage,

125 'startpage': startpage,

@@ -154,5 +136,5 @@ def search_autocomplete(backend_name, query, lang):

154

136

155 try:

137 try:

156 return backend(query, lang)

138 return backend(query, lang)

157− except (HTTPError, SearxEngineResponseException):

139+ except (RequestException, SearxEngineResponseException):

158 return []

140 return []

searx/data/__init__.py+14−39

@@ -1,50 +1,25 @@

1−# SPDX-License-Identifier: AGPL-3.0-or-later

2−# lint: pylint

3−"""This module holds the *data* created by::

4−

5− make data.all

6−

7−"""

8−

9−__all__ = [

10− 'ENGINES_LANGUAGES',

11− 'CURRENCIES',

12− 'USER_AGENTS',

13− 'EXTERNAL_URLS',

14− 'WIKIDATA_UNITS',

15− 'EXTERNAL_BANGS',

16− 'OSM_KEYS_TAGS',

17− 'ahmia_blacklist_loader',

18−]

19−

20import json

1import json

21from pathlib import Path

2from pathlib import Path

22

3

4+

5+__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'EXTERNAL_URLS', 'WIKIDATA_UNITS', 'EXTERNAL_BANGS',

6+ 'bangs_loader', 'ahmia_blacklist_loader']

23data_dir = Path(__file__).parent

7data_dir = Path(__file__).parent

24

8

25

9

26−def _load(filename):

10+def load(filename):

27− with open(data_dir / filename, encoding='utf-8') as f:

11+ with open(data_dir / filename, encoding='utf-8') as fd:

28− return json.load(f)

12+ return json.load(fd)

29

13

30

14

31def ahmia_blacklist_loader():

15def ahmia_blacklist_loader():

32− """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion

33− names. The MD5 values are fetched by::

34−

35− searx_extra/update/update_ahmia_blacklist.py

36−

37− This function is used by :py:mod:`searx.plugins.ahmia_filter`.

38−

39− """

40− with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as f:

16+ with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:

41− return f.read().split()

17+ return fd.read().split()

42

18

43

19

44−ENGINES_LANGUAGES = _load('engines_languages.json')

20+ENGINES_LANGUAGES = load('engines_languages.json')

45−CURRENCIES = _load('currencies.json')

21+CURRENCIES = load('currencies.json')

46−USER_AGENTS = _load('useragents.json')

22+USER_AGENTS = load('useragents.json')

47−EXTERNAL_URLS = _load('external_urls.json')

23+EXTERNAL_URLS = load('external_urls.json')

48−WIKIDATA_UNITS = _load('wikidata_units.json')

24+WIKIDATA_UNITS = load('wikidata_units.json')

49−EXTERNAL_BANGS = _load('external_bangs.json')

25+EXTERNAL_BANGS = load('external_bangs.json')

50−OSM_KEYS_TAGS = _load('osm_keys_tags.json')

searx/data/ahmia_blacklist.txt−10501