3a0f896...master · searx

.github/workflows/data-update.yml+2−2

since splitsnapshots

unifiedsplit

@@ -29,7 +29,7 @@ jobs:

29 - name: Set up Python

30 uses: actions/setup-python@v2

31 with:

32− python-version: '3.9'

32+ python-version: '3.10'

33 architecture: 'x64'

34

35 - name: Install Python dependencies

@@ -40,7 +40,7 @@ jobs:

40 env:

41 FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}

42 run: |

43− V=1 manage pyenv.cmd python "$FETCH_SCRIPT"

43+ V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"

44

45 - name: Create Pull Request

46 id: cpr

.github/workflows/integration.yml+32−3

@@ -13,7 +13,7 @@ jobs:

13 strategy:

14 matrix:

15 os: [ubuntu-20.04]

16− python-version: [3.6, 3.7, 3.8, 3.9]

16+ python-version: [3.7, 3.8, 3.9, "3.10"]

17 steps:

18 - name: Checkout

19 uses: actions/checkout@v2

@@ -56,6 +56,17 @@ jobs:

56 uses: actions/checkout@v2

57 - name: Install Ubuntu packages

58 run: sudo ./utils/searx.sh install packages

59+ - name: Set up Python

60+ uses: actions/setup-python@v2

61+ with:

62+ python-version: '3.9'

63+ architecture: 'x64'

64+ - name: Cache Python dependencies

65+ id: cache-python

66+ uses: actions/cache@v2

67+ with:

68+ path: ./local

69+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

59 - name: Install node dependencies

70 - name: Install node dependencies

60 run: make V=1 node.env

71 run: make V=1 node.env

61 - name: Build themes

72 - name: Build themes

@@ -68,14 +79,21 @@ jobs:

68 - name: Checkout

79 - name: Checkout

69 uses: actions/checkout@v2

80 uses: actions/checkout@v2

70 with:

81 with:

82+ fetch-depth: '0'

71 persist-credentials: false

83 persist-credentials: false

72 - name: Install Ubuntu packages

84 - name: Install Ubuntu packages

73 run: sudo ./utils/searx.sh install buildhost

85 run: sudo ./utils/searx.sh install buildhost

74 - name: Set up Python

86 - name: Set up Python

75 uses: actions/setup-python@v2

87 uses: actions/setup-python@v2

76 with:

88 with:

77− python-version: '3.9'

89+ python-version: '3.10'

78 architecture: 'x64'

90 architecture: 'x64'

91+ - name: Cache Python dependencies

92+ id: cache-python

93+ uses: actions/cache@v2

94+ with:

95+ path: ./local

96+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

79 - name: Build documentation

97 - name: Build documentation

80 run: |

98 run: |

81 make V=1 docs.clean docs.html

99 make V=1 docs.clean docs.html

@@ -97,7 +115,7 @@ jobs:

97 - documentation

115 - documentation

98 env:

116 env:

99 DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

117 DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

100− runs-on: ubuntu-18.04

118+ runs-on: ubuntu-20.04

101 steps:

119 steps:

102 - name: Checkout

120 - name: Checkout

103 if: env.DOCKERHUB_USERNAME != null

121 if: env.DOCKERHUB_USERNAME != null

@@ -105,6 +123,17 @@ jobs:

105 with:

123 with:

106 # make sure "make docker.push" can get the git history

124 # make sure "make docker.push" can get the git history

107 fetch-depth: '0'

125 fetch-depth: '0'

126+ - name: Set up Python

127+ uses: actions/setup-python@v2

128+ with:

129+ python-version: '3.9'

130+ architecture: 'x64'

131+ - name: Cache Python dependencies

132+ id: cache-python

133+ uses: actions/cache@v2

134+ with:

135+ path: ./local

136+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

108 - name: Set up QEMU

137 - name: Set up QEMU

109 if: env.DOCKERHUB_USERNAME != null

138 if: env.DOCKERHUB_USERNAME != null

110 uses: docker/setup-qemu-action@v1

139 uses: docker/setup-qemu-action@v1

.gitignore+2

@@ -26,3 +26,5 @@ dist/

26local/

27gh-pages/

28searx.egg-info/

29+.env

30+geckodriver.log

.pylintrc+1−1

@@ -59,7 +59,7 @@ confidence=

59# --enable=similarities". If you want to run only the classes checker, but have

60# no Warning level messages displayed, use"--disable=all --enable=classes

61# --disable=W"

62−disable=bad-whitespace, duplicate-code

62+disable=bad-whitespace, duplicate-code, consider-using-f-string

63

64# Enable the message, report, category or checker with the given id(s). You can

65# either give multiple identifier separated by comma (,) or put this option

AUTHORS.rst+1

@@ -165,3 +165,4 @@ generally made searx better:

165- @jhigginbotham

166- @xenrox

167- @OliveiraHermogenes

168+- Paul Alcock @Guilvareux

Dockerfile+5−4

@@ -1,4 +1,4 @@

1−FROM alpine:3.12

1+FROM alpine:3.15

2ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

3EXPOSE 8080

4VOLUME /etc/searx

@@ -46,18 +46,19 @@ RUN apk upgrade --no-cache \

46 uwsgi \

47 uwsgi-python3 \

48 brotli \

49− && pip3 install --upgrade pip \

49+ && pip3 install --upgrade pip wheel setuptools \

50 && pip3 install --no-cache -r requirements.txt \

51 && apk del build-dependencies \

52 && rm -rf /root/.cache

53

54−COPY --chown=searx:searx . .

54+COPY searx ./searx

55+COPY dockerfiles ./dockerfiles

55

56

56ARG TIMESTAMP_SETTINGS=0

57ARG TIMESTAMP_SETTINGS=0

57ARG TIMESTAMP_UWSGI=0

58ARG TIMESTAMP_UWSGI=0

58ARG VERSION_GITCOMMIT=unknown

59ARG VERSION_GITCOMMIT=unknown

59

60

60−RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

61+RUN /usr/bin/python3 -m compileall -q searx; \

61 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

62 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

62 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

63 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

63 if [ ! -z $VERSION_GITCOMMIT ]; then\

64 if [ ! -z $VERSION_GITCOMMIT ]; then\

README.rst+2−2

@@ -17,7 +17,7 @@

17|OpenCollective searx backers|

18|OpenCollective searx sponsors|

19

20−Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **səːks**.

20+Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **sɜːks**.

21

22.. _metasearch engine: https://en.wikipedia.org/wiki/Metasearch_engine

23

@@ -61,7 +61,7 @@ our homepage_.

61.. _homepage: https://searx.github.io/searx

62

63contact:

64− openhub_ // twitter_ // IRC: #searx @ freenode

64+ openhub_ // twitter_ // IRC: #searx @ Libera (irc.libera.chat)

65

66.. _openhub: https://www.openhub.net/p/searx

67.. _twitter: https://twitter.com/Searx_engine

docs/admin/settings.rst+51−13

@@ -130,14 +130,12 @@ Global Settings

130 request_timeout : 2.0 # default timeout in seconds, can be override by engine

131 # max_request_timeout: 10.0 # the maximum timeout in seconds

132 useragent_suffix : "" # informations like an email address to the administrator

133− pool_connections : 100 # Number of different hosts

133+ pool_connections : 100 # Maximum number of allowable connections, or None for no limits. The default is 100.

134− pool_maxsize : 10 # Number of simultaneous requests by host

134+ pool_maxsize : 10 # Number of allowable keep-alive connections, or None to always allow. The default is 10.

135+ enable_http2: True # See https://www.python-httpx.org/http2/

135 # uncomment below section if you want to use a proxy

136 # uncomment below section if you want to use a proxy

136 # proxies:

137 # proxies:

137− # http:

138+ # all://:

138− # - http://proxy1:8080

139− # - http://proxy2:8080

140− # https:

141 # - http://proxy1:8080

139 # - http://proxy1:8080

142 # - http://proxy2:8080

140 # - http://proxy2:8080

143 # uncomment below section only if you have more than one network interface

141 # uncomment below section only if you have more than one network interface

@@ -145,6 +143,7 @@ Global Settings

145 # source_ips:

143 # source_ips:

146 # - 1.1.1.1

144 # - 1.1.1.1

147 # - 1.1.1.2

145 # - 1.1.1.2

146+ # - fe80::/126

148

147

149

148

150``request_timeout`` :

149``request_timeout`` :

@@ -157,20 +156,46 @@ Global Settings

157 Suffix to the user-agent searx uses to send requests to others engines. If an

156 Suffix to the user-agent searx uses to send requests to others engines. If an

158 engine wish to block you, a contact info here may be useful to avoid that.

157 engine wish to block you, a contact info here may be useful to avoid that.

159

158

160−.. _requests proxies: https://requests.readthedocs.io/en/latest/user/advanced/#proxies

159+``keepalive_expiry``:

161−.. _PySocks: https://pypi.org/project/PySocks/

160+ Number of seconds to keep a connection in the pool. By default 5.0 seconds.

161+

162+.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying

162

163

163``proxies`` :

164``proxies`` :

164− Define one or more proxies you wish to use, see `requests proxies`_.

165+ Define one or more proxies you wish to use, see `httpx proxies`_.

165 If there are more than one proxy for one protocol (http, https),

166 If there are more than one proxy for one protocol (http, https),

166 requests to the engines are distributed in a round-robin fashion.

167 requests to the engines are distributed in a round-robin fashion.

167

168

168− - Proxy: `see <https://2.python-requests.org/en/latest/user/advanced/#proxies>`__.

169− - SOCKS proxies are also supported: `see <https://2.python-requests.org/en/latest/user/advanced/#socks>`__

170−

171``source_ips`` :

169``source_ips`` :

172 If you use multiple network interfaces, define from which IP the requests must

170 If you use multiple network interfaces, define from which IP the requests must

173− be made. This parameter is ignored when ``proxies`` is set.

171+ be made. Example:

172+

173+ * ``0.0.0.0`` any local IPv4 address.

174+ * ``::`` any local IPv6 address.

175+ * ``192.168.0.1``

176+ * ``[ 192.168.0.1, 192.168.0.2 ]`` these two specific IP addresses

177+ * ``fe80::60a2:1691:e5a2:ee1f``

178+ * ``fe80::60a2:1691:e5a2:ee1f/126`` all IP addresses in this network.

179+ * ``[ 192.168.0.1, fe80::/126 ]``

180+

181+``retries`` :

182+ Number of retry in case of an HTTP error.

183+ On each retry, searx uses an different proxy and source ip.

184+

185+``retry_on_http_error`` :

186+ Retry request on some HTTP status code.

187+

188+ Example:

189+

190+ * ``true`` : on HTTP status code between 400 and 599.

191+ * ``403`` : on HTTP status code 403.

192+ * ``[403, 429]``: on HTTP status code 403 and 429.

193+

194+``enable_http2`` :

195+ Enable by default. Set to ``False`` to disable HTTP/2.

196+

197+``max_redirects`` :

198+ 30 by default. Maximum redirect before it is an error.

174

199

175

200

176``locales:``

201``locales:``

@@ -216,6 +241,13 @@ Engine settings

216 api_key : 'apikey'

241 api_key : 'apikey'

217 disabled : True

242 disabled : True

218 language : en_US

243 language : en_US

244+ #enable_http: False

245+ #enable_http2: False

246+ #retries: 1

247+ #retry_on_http_error: True # or 403 or [404, 429]

248+ #max_connections: 100

249+ #max_keepalive_connections: 10

250+ #keepalive_expiry: 5.0

219 #proxies:

251 #proxies:

220 # http:

252 # http:

221 # - http://proxy1:8080

253 # - http://proxy1:8080

@@ -270,6 +302,12 @@ Engine settings

270``display_error_messages`` : default ``True``

302``display_error_messages`` : default ``True``

271 When an engine returns an error, the message is displayed on the user interface.

303 When an engine returns an error, the message is displayed on the user interface.

272

304

305+``network``: optional

306+ Use the network configuration from another engine.

307+ In addition, there are two default networks:

308+ * ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses)

309+ * ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses)

310+

273.. note::

311.. note::

274

312

275 A few more options are possible, but they are pretty specific to some

313 A few more options are possible, but they are pretty specific to some

docs/admin/update-searx.rst+2−2

@@ -23,7 +23,7 @@ How to update depends on the :ref:`installation` method. If you have used the

23

24.. code:: bash

25

26− $ sudo -H ./utils/morty.sh update morty

26+ sudo -H ./utils/morty.sh update morty

27

28.. _inspect searx:

29

@@ -55,5 +55,5 @@ How to debug depends on the :ref:`installation` method. If you have used the

55

56.. code:: bash

57

58− $ sudo -H ./utils/morty.sh inspect service

58+ sudo -H ./utils/morty.sh inspect service

59

docs/blog/index.rst+5−3

@@ -3,13 +3,15 @@ Blog

3====

4

5.. toctree::

6− :maxdepth: 2

6+ :titlesonly:

7− :caption: Contents

7+ :reversed:

8

9− lxcdev-202006

10 python3

9 python3

11 admin

10 admin

12 intro-offline

11 intro-offline

13 private-engines

12 private-engines

13+ lxcdev-202006

14 command-line-engines

15 search-indexer-engines

16+ sql-engines

17+ search-database-engines

docs/blog/search-database-engines.rstnew file+95

@@ -0,0 +1,95 @@

1+===============================

2+Query more of your NoSQL stores

3+===============================

4+

5+From now on, searx lets you to query your NoSQL data stores:

6+

7+* `Redis`_

8+* `MongoDB`_

9+

10+The reference configuration of the engines are included ``settings.yml`` just commented out,

11+as you have to set various options and install dependencies before using them.

12+

13+By default, the engines use ``key-value`` template for displaying results.

14+If you are not satisfied with the original result layout,

15+you can use your owm template by placing the template under

16+``searx/templates/{theme_name}/result_templates/{template_name}`` and setting

17+``result_template`` attribute to ``{template_name}``.

18+

19+Futhermore, if you do not want to expose these engines on a public instance, you can

20+still add them and limit the access by setting ``tokens`` as described in the `blog post about

21+private engines`_.

22+

23+Configuring searx to use the stores

24+===================================

25+

26+NoSQL data stores are used for storing arbitrary data without first defining their

27+structure.

28+

29+Redis

30+-----

31+

32+Reqired package: ``redis``

33+

34+Redis is a key value based data store usually stored in memory.

35+

36+Select a database to search in and set its index in the option ``db``. You can

37+either look for exact matches or use partial keywords to find what you are looking for

38+by configuring ``exact_match_only``.

39+

40+In this example you can search for exact matches in your first database:

41+

42+.. code:: yaml

43+

44+ - name : mystore

45+ engine : redis_server

46+ exact_match_only : True

47+ host : 127.0.0.1

48+ port : 6379

49+ password : secret-password

50+ db : 0

51+ shortcut : rds

52+ enable_http : True

53+

54+

55+MongoDB

56+-------

57+

58+Required package: ``pymongo``

59+

60+MongoDB is a document based database program that handles JSON like data.

61+

62+In order to query MongoDB, you have to select a ``database`` and a ``collection``. Furthermore,

63+you have to select a ``key`` that is going to be searched. MongoDB also supports the option ``exact_match_only``, so configure it

64+as you wish.

65+

66+Above is an example configuration for using a MongoDB collection:

67+

68+.. code:: yaml

69+

70+ - name : mymongo

71+ engine : mongodb

72+ shortcut : md

73+ host : '127.0.0.1'

74+ port : 27017

75+ database : personal

76+ collection : income

77+ key : month

78+ enable_http: True

79+

80+

81+Acknowledgement

82+===============

83+

84+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .

85+

86+.. _Redis: https://redis.io/

87+.. _MongoDB: https://mongodb.com/

88+.. _blog post about private engines: private-engines.html#private-engines

89+.. _Search and Discovery Fund: https://nlnet.nl/discovery

90+.. _NLnet Foundation: https://nlnet.nl/

91+

92+

93+| Happy hacking.

94+| kvch // 2021.07.13 23:16

95+

docs/blog/sql-engines.rstnew file+117

@@ -0,0 +1,117 @@

1+=================

2+Query SQL servers

3+=================

4+

5+Now you can query SQL servers using searx. The following ones are supported:

6+

7+* `PostgreSQL`_

8+* `MySQL`_

9+* `SQLite`_

10+

11+All of the engines above are added to ``settings.yml`` just commented out, as you have to

12+set the required attributes for the engines, e.g. ``database``. By default, the engines use

13+``key-value`` template for displaying results. If you are not satisfied with the original result layout,

14+you can use your owm template by placing the template under

15+``searx/templates/{theme_name}/result_templates/{template_name}`` and setting

16+``result_template`` attribute to ``{template_name}``.

17+

18+As mentioned in previous blog posts, if you do not wish to expose these engines on a

19+public instance, you can still add them and limit the access by setting ``tokens``

20+as described in the `blog post about private engines`_.

21+

22+Configure the engines

23+=====================

24+

25+The configuration of the new database engines are similar. You must put a valid

26+SELECT SQL query in ``query_str``. At the moment you can only bind at most

27+one parameter in your query. By setting the attribute ``limit`` you can

28+define how many results you want from the SQL server. Basically, it

29+is the same as the LIMIT keyword in SQL.

30+

31+Please, do not include LIMIT or OFFSET in your SQL query as the engines

32+rely on these keywords during paging. If you want to configure the number of returned results

33+use the option ``limit``.

34+

35+PostgreSQL

36+----------

37+

38+PostgreSQL is a powerful and robust open source database.

39+

40+Before configuring the PostgreSQL engine, you must install the dependency ``psychopg2``.

41+

42+You can find an example configuration below:

43+

44+.. code:: yaml

45+

46+ - name : postgresql

47+ engine : postgresql

48+ database : my_database

49+ username : searx

50+ password : password

51+ query_str : 'SELECT * from my_table WHERE my_column = %(query)s'

52+ shortcut : psql

53+

54+

55+MySQL

56+-----

57+

58+MySQL is said to be the most popular open source database.

59+

60+Before enabling MySQL engine, you must install the package ``mysql-connector-python``.

61+

62+The authentication plugin is configurable by setting ``auth_plugin`` in the attributes.

63+By default it is set to ``caching_sha2_password``.

64+

65+This is an example configuration for quering a MySQL server:

66+

67+.. code:: yaml

68+

69+ - name : mysql

70+ engine : mysql_server

71+ database : my_database

72+ username : searx

73+ password : password

74+ limit : 5

75+ query_str : 'SELECT * from my_table WHERE my_column=%(query)s'

76+ shortcut : mysql

77+

78+

79+SQLite

80+------

81+

82+SQLite is a small, fast and reliable SQL database engine. It does not require

83+any extra dependency.

84+

85+You can read from your database ``my_database`` using this example configuration:

86+

87+.. code:: yaml

88+

89+ - name : sqlite

90+ engine : sqlite

91+ shortcut: sq

92+ database : my_database

93+ query_str : 'SELECT * FROM my_table WHERE my_column=:query'

94+

95+

96+Next steps

97+==========

98+

99+The next step is to add support for more data stores, e.g. Redis and MongoDB.

100+

101+Acknowledgement

102+===============

103+

104+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .

105+

106+.. _PostgreSQL: https://www.postgresql.org/

107+.. _MySQL: https://www.mysql.com/

108+.. _SQLite: https://www.sqlite.org/index.html

109+.. _blog post about private engines: private-engines.html#private-engines

110+.. _Search and Discovery Fund: https://nlnet.nl/discovery

111+.. _NLnet Foundation: https://nlnet.nl/

112+

113+

114+| Happy hacking.

115+| kvch // 2021.05.23 23:16

116+

117+

docs/conf.py+2−1

@@ -10,7 +10,7 @@ from searx.version import VERSION_STRING

10# Project --------------------------------------------------------------

11

12project = u'searx'

13−copyright = u'2015-2020, Adam Tauber, Noémi Ványi'

13+copyright = u'2015-2021, Adam Tauber, Noémi Ványi'

14author = u'Adam Tauber'

15release, version = VERSION_STRING, VERSION_STRING

16highlight_language = 'none'

@@ -101,6 +101,7 @@ imgmath_font_size = 14

101

102html_theme_options = {"index_sidebar_logo": True}

103html_context = {"project_links": [] }

104+html_context["project_links"].append(ProjectLink("Blog", "blog/index.html"))

104if brand.GIT_URL:

105if brand.GIT_URL:

105 html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))

106 html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))

106if brand.WIKI_URL:

107if brand.WIKI_URL:

docs/dev/engine_overview.rst+1−1

@@ -171,7 +171,7 @@ headers set HTTP header information

171data set HTTP data information

172cookies set HTTP cookies

173verify bool Performing SSL-Validity check

174−allow_redirects bool Follow redirects

174+follow_redirects bool Follow redirects

175max_redirects int maximum redirects, hard limit

176soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

177raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300

manage+6−5

@@ -13,7 +13,7 @@ source_dot_config

13

14PY_SETUP_EXTRAS='[test]'

15NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"

16−GECKODRIVER_VERSION="v0.28.0"

16+GECKODRIVER_VERSION="v0.30.0"

17# SPHINXOPTS=

18

19# These py files are linted by test.pylint(), all other files are linted by

@@ -32,6 +32,7 @@ PYLINT_FILES=(

32 searx/engines/meilisearch.py

33 searx/engines/solidtorrents.py

34 searx/engines/solr.py

35+ searx/engines/sqlite.py

35 searx/engines/springer.py

36 searx/engines/springer.py

36 searx/engines/google_scholar.py

37 searx/engines/google_scholar.py

37 searx/engines/yahoo_news.py

38 searx/engines/yahoo_news.py

@@ -223,7 +224,7 @@ docker.build() {

223

224

224 # build Docker image

225 # build Docker image

225 build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

226 build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

226− sudo docker build \

227+ docker build \

227 --build-arg GIT_URL="${GIT_URL}" \

228 --build-arg GIT_URL="${GIT_URL}" \

228 --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

229 --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

229 --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

230 --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

@@ -235,8 +236,8 @@ docker.build() {

235 -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

236 -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

236

237

237 if [ "$1" = "push" ]; then

238 if [ "$1" = "push" ]; then

238− sudo docker push "${SEARX_IMAGE_NAME}:latest"

239+ docker push "${SEARX_IMAGE_NAME}:latest"

239− sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

240+ docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

240 fi

241 fi

241 )

242 )

242 dump_return $?

243 dump_return $?

@@ -399,7 +400,7 @@ test.pylint() {

399 pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

400 pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

400 --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

401 --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

401 --ignore=searx/engines \

402 --ignore=searx/engines \

402− searx tests

403+ tests

403 )

404 )

404 dump_return $?

405 dump_return $?

405}

406}

requirements-dev.txt+14−12

@@ -1,18 +1,20 @@

1mock==4.0.3

2nose2[coverage_plugin]==0.10.0

3cov-core==1.15.0

4−pycodestyle==2.7.0

4+pycodestyle==2.8.0

5−pylint==2.7.4

5+pylint==2.12.2

6−splinter==0.14.0

6+splinter==0.17.0

7−transifex-client==0.14.2

7+transifex-client==0.14.3; python_version < '3.10'

8+transifex-client==0.12.1; python_version == '3.10'

8−selenium==3.141.0

9+selenium==4.1.0

9−twine==3.4.1

10+twine==3.7.1

10−Pallets-Sphinx-Themes==1.2.3

11+Pallets-Sphinx-Themes==2.0.2

11docutils==0.16

12docutils==0.16

12−Sphinx==3.5.4

13+Sphinx==4.4.0

13−sphinx-issues==1.2.0

14+sphinx-issues==3.0.1

14−sphinx-jinja==1.1.1

15+sphinx-jinja==1.4.0

15−sphinx-tabs==2.1.0

16+sphinx-tabs==3.2.0

16sphinxcontrib-programoutput==0.17

17sphinxcontrib-programoutput==0.17

17sphinx-autobuild==2021.3.14

18sphinx-autobuild==2021.3.14

18−linuxdoc==20210324

19+linuxdoc==20211220

20+aiounittest==1.4.1

requirements.txt+14−10

@@ -1,12 +1,16 @@

1−certifi==2020.12.05

1+certifi==2021.10.8

2−babel==2.9.0

2+babel==2.9.1

3flask-babel==2.0.0

4−flask==1.1.2

4+flask==2.0.2

5−idna==2.10

6−jinja2==2.11.3

5+jinja2==3.0.3

7−lxml==4.6.3

6+lxml==4.7.1

8pygments==2.8.0

7pygments==2.8.0

9−python-dateutil==2.8.1

8+python-dateutil==2.8.2

10−pyyaml==5.4.1

9+pyyaml==6.0

11−requests[socks]==2.25.1

10+httpx[http2]==0.21.3

11+Brotli==1.0.9

12+uvloop==0.16.0; python_version >= '3.7'

13+uvloop==0.14.0; python_version < '3.7'

14+httpx-socks[asyncio]==0.7.2

12−langdetect==1.0.8

15+langdetect==1.0.9

16+setproctitle==1.2.2

searx/autocomplete.py+28−10

@@ -20,10 +20,12 @@ from lxml import etree

20from json import loads

21from urllib.parse import urlencode

22

23−from requests import RequestException

23+from httpx import HTTPError

24+

24

25

25from searx import settings

26from searx import settings

27+from searx.data import ENGINES_LANGUAGES

26−from searx.poolrequests import get as http_get

28+from searx.network import get as http_get

27from searx.exceptions import SearxEngineResponseException

29from searx.exceptions import SearxEngineResponseException

28

30

29

31

@@ -34,6 +36,22 @@ def get(*args, **kwargs):

34 return http_get(*args, **kwargs)

36 return http_get(*args, **kwargs)

35

37

36

38

39+def brave(query, lang):

40+ # brave search autocompleter

41+ url = 'https://search.brave.com/api/suggest?{query}'

42+

43+ resp = get(url.format(query=urlencode({'q': query})))

44+

45+ results = []

46+

47+ if resp.ok:

48+ data = loads(resp.text)

49+ for suggestion in data[1]:

50+ results.append(suggestion)

51+

52+ return results

53+

54+

37def dbpedia(query, lang):

55def dbpedia(query, lang):

38 # dbpedia autocompleter, no HTTPS

56 # dbpedia autocompleter, no HTTPS

39 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

57 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

@@ -76,12 +94,11 @@ def google(query, lang):

76

94

77def startpage(query, lang):

95def startpage(query, lang):

78 # startpage autocompleter

96 # startpage autocompleter

97+ lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')

79− url = 'https://startpage.com/do/suggest?{query}'

98+ url = 'https://startpage.com/suggestions?{query}'

80−

81− resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')

99+ resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))

82− if len(resp) > 1:

100+ data = resp.json()

83− return resp

101+ return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

84− return []

85

102

86

103

87def swisscows(query, lang):

104def swisscows(query, lang):

@@ -119,7 +136,8 @@ def wikipedia(query, lang):

119 return []

136 return []

120

137

121

138

139+backends = {'brave': brave,

122−backends = {'dbpedia': dbpedia,

140+ 'dbpedia': dbpedia,

123 'duckduckgo': duckduckgo,

141 'duckduckgo': duckduckgo,

124 'google': google,

142 'google': google,

125 'startpage': startpage,

143 'startpage': startpage,

@@ -136,5 +154,5 @@ def search_autocomplete(backend_name, query, lang):

136

154

137 try:

155 try:

138 return backend(query, lang)

156 return backend(query, lang)

139− except (RequestException, SearxEngineResponseException):

157+ except (HTTPError, SearxEngineResponseException):

140 return []

158 return []

searx/data/__init__.py+39−14

@@ -1,25 +1,50 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+# lint: pylint

3+"""This module holds the *data* created by::

4+

5+ make data.all

6+

7+"""

8+

9+__all__ = [

10+ 'ENGINES_LANGUAGES',

11+ 'CURRENCIES',

12+ 'USER_AGENTS',

13+ 'EXTERNAL_URLS',

14+ 'WIKIDATA_UNITS',

15+ 'EXTERNAL_BANGS',

16+ 'OSM_KEYS_TAGS',

17+ 'ahmia_blacklist_loader',

18+]

19+

1import json

20import json

2from pathlib import Path

21from pathlib import Path

3

22

4−

5−__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'EXTERNAL_URLS', 'WIKIDATA_UNITS', 'EXTERNAL_BANGS',

6− 'bangs_loader', 'ahmia_blacklist_loader']

7data_dir = Path(__file__).parent

23data_dir = Path(__file__).parent

8

24

9

25

10−def load(filename):

26+def _load(filename):

11− with open(data_dir / filename, encoding='utf-8') as fd:

27+ with open(data_dir / filename, encoding='utf-8') as f:

12− return json.load(fd)

28+ return json.load(f)

13

29

14

30

15def ahmia_blacklist_loader():

31def ahmia_blacklist_loader():

32+ """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion

33+ names. The MD5 values are fetched by::

34+

35+ searx_extra/update/update_ahmia_blacklist.py

36+

37+ This function is used by :py:mod:`searx.plugins.ahmia_filter`.

38+

39+ """

16− with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:

40+ with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as f:

17− return fd.read().split()

41+ return f.read().split()

18

42

19

43

20−ENGINES_LANGUAGES = load('engines_languages.json')

44+ENGINES_LANGUAGES = _load('engines_languages.json')

21−CURRENCIES = load('currencies.json')

45+CURRENCIES = _load('currencies.json')

22−USER_AGENTS = load('useragents.json')

46+USER_AGENTS = _load('useragents.json')

23−EXTERNAL_URLS = load('external_urls.json')

47+EXTERNAL_URLS = _load('external_urls.json')

24−WIKIDATA_UNITS = load('wikidata_units.json')

48+WIKIDATA_UNITS = _load('wikidata_units.json')

25−EXTERNAL_BANGS = load('external_bangs.json')

49+EXTERNAL_BANGS = _load('external_bangs.json')

50+OSM_KEYS_TAGS = _load('osm_keys_tags.json')

searx/data/ahmia_blacklist.txt+11140