66206bf...master · searx

.github/workflows/data-update.yml+1−1

since splitsnapshots

unifiedsplit

@@ -29,7 +29,7 @@ jobs:

29 - name: Set up Python

30 uses: actions/setup-python@v2

31 with:

32− python-version: '3.9'

32+ python-version: '3.10'

33 architecture: 'x64'

34

35 - name: Install Python dependencies

.github/workflows/integration.yml+32−3

@@ -13,7 +13,7 @@ jobs:

13 strategy:

14 matrix:

15 os: [ubuntu-20.04]

16− python-version: [3.6, 3.7, 3.8, 3.9]

16+ python-version: [3.7, 3.8, 3.9, "3.10"]

17 steps:

18 - name: Checkout

19 uses: actions/checkout@v2

@@ -56,6 +56,17 @@ jobs:

56 uses: actions/checkout@v2

57 - name: Install Ubuntu packages

58 run: sudo ./utils/searx.sh install packages

59+ - name: Set up Python

60+ uses: actions/setup-python@v2

61+ with:

62+ python-version: '3.9'

63+ architecture: 'x64'

64+ - name: Cache Python dependencies

65+ id: cache-python

66+ uses: actions/cache@v2

67+ with:

68+ path: ./local

69+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

59 - name: Install node dependencies

70 - name: Install node dependencies

60 run: make V=1 node.env

71 run: make V=1 node.env

61 - name: Build themes

72 - name: Build themes

@@ -68,14 +79,21 @@ jobs:

68 - name: Checkout

79 - name: Checkout

69 uses: actions/checkout@v2

80 uses: actions/checkout@v2

70 with:

81 with:

82+ fetch-depth: '0'

71 persist-credentials: false

83 persist-credentials: false

72 - name: Install Ubuntu packages

84 - name: Install Ubuntu packages

73 run: sudo ./utils/searx.sh install buildhost

85 run: sudo ./utils/searx.sh install buildhost

74 - name: Set up Python

86 - name: Set up Python

75 uses: actions/setup-python@v2

87 uses: actions/setup-python@v2

76 with:

88 with:

77− python-version: '3.9'

89+ python-version: '3.10'

78 architecture: 'x64'

90 architecture: 'x64'

91+ - name: Cache Python dependencies

92+ id: cache-python

93+ uses: actions/cache@v2

94+ with:

95+ path: ./local

96+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

79 - name: Build documentation

97 - name: Build documentation

80 run: |

98 run: |

81 make V=1 docs.clean docs.html

99 make V=1 docs.clean docs.html

@@ -97,7 +115,7 @@ jobs:

97 - documentation

115 - documentation

98 env:

116 env:

99 DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

117 DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

100− runs-on: ubuntu-18.04

118+ runs-on: ubuntu-20.04

101 steps:

119 steps:

102 - name: Checkout

120 - name: Checkout

103 if: env.DOCKERHUB_USERNAME != null

121 if: env.DOCKERHUB_USERNAME != null

@@ -105,6 +123,17 @@ jobs:

105 with:

123 with:

106 # make sure "make docker.push" can get the git history

124 # make sure "make docker.push" can get the git history

107 fetch-depth: '0'

125 fetch-depth: '0'

126+ - name: Set up Python

127+ uses: actions/setup-python@v2

128+ with:

129+ python-version: '3.9'

130+ architecture: 'x64'

131+ - name: Cache Python dependencies

132+ id: cache-python

133+ uses: actions/cache@v2

134+ with:

135+ path: ./local

136+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

108 - name: Set up QEMU

137 - name: Set up QEMU

109 if: env.DOCKERHUB_USERNAME != null

138 if: env.DOCKERHUB_USERNAME != null

110 uses: docker/setup-qemu-action@v1

139 uses: docker/setup-qemu-action@v1

.gitignore+2

@@ -26,3 +26,5 @@ dist/

26local/

27gh-pages/

28searx.egg-info/

29+.env

30+geckodriver.log

.pylintrc+1−1

@@ -59,7 +59,7 @@ confidence=

59# --enable=similarities". If you want to run only the classes checker, but have

60# no Warning level messages displayed, use"--disable=all --enable=classes

61# --disable=W"

62−disable=bad-whitespace, duplicate-code

62+disable=bad-whitespace, duplicate-code, consider-using-f-string

63

64# Enable the message, report, category or checker with the given id(s). You can

65# either give multiple identifier separated by comma (,) or put this option

AUTHORS.rst+1

@@ -165,3 +165,4 @@ generally made searx better:

165- @jhigginbotham

166- @xenrox

167- @OliveiraHermogenes

168+- Paul Alcock @Guilvareux

Dockerfile+5−4

@@ -1,4 +1,4 @@

1−FROM alpine:3.12

1+FROM alpine:3.15

2ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

3EXPOSE 8080

4VOLUME /etc/searx

@@ -46,18 +46,19 @@ RUN apk upgrade --no-cache \

46 uwsgi \

47 uwsgi-python3 \

48 brotli \

49− && pip3 install --upgrade pip \

49+ && pip3 install --upgrade pip wheel setuptools \

50 && pip3 install --no-cache -r requirements.txt \

51 && apk del build-dependencies \

52 && rm -rf /root/.cache

53

54−COPY --chown=searx:searx . .

54+COPY searx ./searx

55+COPY dockerfiles ./dockerfiles

55

56

56ARG TIMESTAMP_SETTINGS=0

57ARG TIMESTAMP_SETTINGS=0

57ARG TIMESTAMP_UWSGI=0

58ARG TIMESTAMP_UWSGI=0

58ARG VERSION_GITCOMMIT=unknown

59ARG VERSION_GITCOMMIT=unknown

59

60

60−RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

61+RUN /usr/bin/python3 -m compileall -q searx; \

61 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

62 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

62 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

63 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

63 if [ ! -z $VERSION_GITCOMMIT ]; then\

64 if [ ! -z $VERSION_GITCOMMIT ]; then\

README.rst+2−2

@@ -17,7 +17,7 @@

17|OpenCollective searx backers|

18|OpenCollective searx sponsors|

19

20−Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **səːks**.

20+Privacy-respecting, hackable `metasearch engine`_ / *pronunciation* **sɜːks**.

21

22.. _metasearch engine: https://en.wikipedia.org/wiki/Metasearch_engine

23

@@ -61,7 +61,7 @@ our homepage_.

61.. _homepage: https://searx.github.io/searx

62

63contact:

64− openhub_ // twitter_ // IRC: #searx @ freenode

64+ openhub_ // twitter_ // IRC: #searx @ Libera (irc.libera.chat)

65

66.. _openhub: https://www.openhub.net/p/searx

67.. _twitter: https://twitter.com/Searx_engine

docs/admin/update-searx.rst+2−2

@@ -23,7 +23,7 @@ How to update depends on the :ref:`installation` method. If you have used the

23

24.. code:: bash

25

26− $ sudo -H ./utils/morty.sh update morty

26+ sudo -H ./utils/morty.sh update morty

27

28.. _inspect searx:

29

@@ -55,5 +55,5 @@ How to debug depends on the :ref:`installation` method. If you have used the

55

56.. code:: bash

57

58− $ sudo -H ./utils/morty.sh inspect service

58+ sudo -H ./utils/morty.sh inspect service

59

docs/blog/index.rst+4−3

@@ -3,14 +3,15 @@ Blog

3====

4

5.. toctree::

6− :maxdepth: 2

6+ :titlesonly:

7− :caption: Contents

7+ :reversed:

8

9− lxcdev-202006

10 python3

9 python3

11 admin

10 admin

12 intro-offline

11 intro-offline

13 private-engines

12 private-engines

13+ lxcdev-202006

14 command-line-engines

15 search-indexer-engines

16 sql-engines

17+ search-database-engines

docs/blog/search-database-engines.rstnew file+95

@@ -0,0 +1,95 @@

1+===============================

2+Query more of your NoSQL stores

3+===============================

4+

5+From now on, searx lets you to query your NoSQL data stores:

6+

7+* `Redis`_

8+* `MongoDB`_

9+

10+The reference configuration of the engines are included ``settings.yml`` just commented out,

11+as you have to set various options and install dependencies before using them.

12+

13+By default, the engines use ``key-value`` template for displaying results.

14+If you are not satisfied with the original result layout,

15+you can use your owm template by placing the template under

16+``searx/templates/{theme_name}/result_templates/{template_name}`` and setting

17+``result_template`` attribute to ``{template_name}``.

18+

19+Futhermore, if you do not want to expose these engines on a public instance, you can

20+still add them and limit the access by setting ``tokens`` as described in the `blog post about

21+private engines`_.

22+

23+Configuring searx to use the stores

24+===================================

25+

26+NoSQL data stores are used for storing arbitrary data without first defining their

27+structure.

28+

29+Redis

30+-----

31+

32+Reqired package: ``redis``

33+

34+Redis is a key value based data store usually stored in memory.

35+

36+Select a database to search in and set its index in the option ``db``. You can

37+either look for exact matches or use partial keywords to find what you are looking for

38+by configuring ``exact_match_only``.

39+

40+In this example you can search for exact matches in your first database:

41+

42+.. code:: yaml

43+

44+ - name : mystore

45+ engine : redis_server

46+ exact_match_only : True

47+ host : 127.0.0.1

48+ port : 6379

49+ password : secret-password

50+ db : 0

51+ shortcut : rds

52+ enable_http : True

53+

54+

55+MongoDB

56+-------

57+

58+Required package: ``pymongo``

59+

60+MongoDB is a document based database program that handles JSON like data.

61+

62+In order to query MongoDB, you have to select a ``database`` and a ``collection``. Furthermore,

63+you have to select a ``key`` that is going to be searched. MongoDB also supports the option ``exact_match_only``, so configure it

64+as you wish.

65+

66+Above is an example configuration for using a MongoDB collection:

67+

68+.. code:: yaml

69+

70+ - name : mymongo

71+ engine : mongodb

72+ shortcut : md

73+ host : '127.0.0.1'

74+ port : 27017

75+ database : personal

76+ collection : income

77+ key : month

78+ enable_http: True

79+

80+

81+Acknowledgement

82+===============

83+

84+This development was sponsored by `Search and Discovery Fund`_ of `NLnet Foundation`_ .

85+

86+.. _Redis: https://redis.io/

87+.. _MongoDB: https://mongodb.com/

88+.. _blog post about private engines: private-engines.html#private-engines

89+.. _Search and Discovery Fund: https://nlnet.nl/discovery

90+.. _NLnet Foundation: https://nlnet.nl/

91+

92+

93+| Happy hacking.

94+| kvch // 2021.07.13 23:16

95+

docs/conf.py+2−1

@@ -10,7 +10,7 @@ from searx.version import VERSION_STRING

10# Project --------------------------------------------------------------

11

12project = u'searx'

13−copyright = u'2015-2020, Adam Tauber, Noémi Ványi'

13+copyright = u'2015-2021, Adam Tauber, Noémi Ványi'

14author = u'Adam Tauber'

15release, version = VERSION_STRING, VERSION_STRING

16highlight_language = 'none'

@@ -101,6 +101,7 @@ imgmath_font_size = 14

101

102html_theme_options = {"index_sidebar_logo": True}

103html_context = {"project_links": [] }

104+html_context["project_links"].append(ProjectLink("Blog", "blog/index.html"))

104if brand.GIT_URL:

105if brand.GIT_URL:

105 html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))

106 html_context["project_links"].append(ProjectLink("Source", brand.GIT_URL))

106if brand.WIKI_URL:

107if brand.WIKI_URL:

docs/dev/engine_overview.rst+1−1

@@ -171,7 +171,7 @@ headers set HTTP header information

171data set HTTP data information

172cookies set HTTP cookies

173verify bool Performing SSL-Validity check

174−allow_redirects bool Follow redirects

174+follow_redirects bool Follow redirects

175max_redirects int maximum redirects, hard limit

176soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

177raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300

manage+4−4

@@ -13,7 +13,7 @@ source_dot_config

13

14PY_SETUP_EXTRAS='[test]'

15NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"

16−GECKODRIVER_VERSION="v0.28.0"

16+GECKODRIVER_VERSION="v0.30.0"

17# SPHINXOPTS=

18

19# These py files are linted by test.pylint(), all other files are linted by

@@ -224,7 +224,7 @@ docker.build() {

224

225 # build Docker image

226 build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

227− sudo docker build \

227+ docker build \

228 --build-arg GIT_URL="${GIT_URL}" \

229 --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

230 --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

@@ -236,8 +236,8 @@ docker.build() {

236 -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

237

238 if [ "$1" = "push" ]; then

239− sudo docker push "${SEARX_IMAGE_NAME}:latest"

239+ docker push "${SEARX_IMAGE_NAME}:latest"

240− sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

240+ docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

241 fi

242 )

243 dump_return $?

requirements-dev.txt+14−13

@@ -1,19 +1,20 @@

1mock==4.0.3

2nose2[coverage_plugin]==0.10.0

3cov-core==1.15.0

4−pycodestyle==2.7.0

4+pycodestyle==2.8.0

5−pylint==2.8.2

5+pylint==2.12.2

6−splinter==0.14.0

6+splinter==0.17.0

7−transifex-client==0.14.2

7+transifex-client==0.14.3; python_version < '3.10'

8+transifex-client==0.12.1; python_version == '3.10'

8−selenium==3.141.0

9+selenium==4.1.0

9−twine==3.4.1

10+twine==3.7.1

10−Pallets-Sphinx-Themes==1.2.3

11+Pallets-Sphinx-Themes==2.0.2

11docutils==0.16

12docutils==0.16

12−Sphinx==3.5.4

13+Sphinx==4.4.0

13−sphinx-issues==1.2.0

14+sphinx-issues==3.0.1

14−sphinx-jinja==1.1.1

15+sphinx-jinja==1.4.0

15−sphinx-tabs==2.1.0

16+sphinx-tabs==3.2.0

16sphinxcontrib-programoutput==0.17

17sphinxcontrib-programoutput==0.17

17sphinx-autobuild==2021.3.14

18sphinx-autobuild==2021.3.14

18−linuxdoc==20210324

19+linuxdoc==20211220

19−aiounittest==1.4.0

20+aiounittest==1.4.1

requirements.txt+10−10

@@ -1,16 +1,16 @@

1−certifi==2020.12.05

1+certifi==2021.10.8

2babel==2.9.1

3flask-babel==2.0.0

4−flask==1.1.2

4+flask==2.0.2

5−jinja2==2.11.3

5+jinja2==3.0.3

6−lxml==4.6.3

6+lxml==4.7.1

7pygments==2.8.0

8−python-dateutil==2.8.1

8+python-dateutil==2.8.2

9−pyyaml==5.4.1

9+pyyaml==6.0

10−httpx[http2]==0.17.1

10+httpx[http2]==0.21.3

11Brotli==1.0.9

12−uvloop==0.15.2; python_version >= '3.7'

12+uvloop==0.16.0; python_version >= '3.7'

13uvloop==0.14.0; python_version < '3.7'

14−httpx-socks[asyncio]==0.3.1

14+httpx-socks[asyncio]==0.7.2

15−langdetect==1.0.8

15+langdetect==1.0.9

16setproctitle==1.2.2

searx/autocomplete.py+24−7

@@ -24,6 +24,7 @@ from httpx import HTTPError

24

25

26from searx import settings

27+from searx.data import ENGINES_LANGUAGES

27from searx.network import get as http_get

28from searx.network import get as http_get

28from searx.exceptions import SearxEngineResponseException

29from searx.exceptions import SearxEngineResponseException

29

30

@@ -35,6 +36,22 @@ def get(*args, **kwargs):

35 return http_get(*args, **kwargs)

36 return http_get(*args, **kwargs)

36

37

38

39+def brave(query, lang):

40+ # brave search autocompleter

41+ url = 'https://search.brave.com/api/suggest?{query}'

42+

43+ resp = get(url.format(query=urlencode({'q': query})))

44+

45+ results = []

46+

47+ if resp.ok:

48+ data = loads(resp.text)

49+ for suggestion in data[1]:

50+ results.append(suggestion)

51+

52+ return results

53+

54+

38def dbpedia(query, lang):

55def dbpedia(query, lang):

39 # dbpedia autocompleter, no HTTPS

56 # dbpedia autocompleter, no HTTPS

40 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

57 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

@@ -77,12 +94,11 @@ def google(query, lang):

77

94

78def startpage(query, lang):

95def startpage(query, lang):

79 # startpage autocompleter

96 # startpage autocompleter

97+ lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')

80− url = 'https://startpage.com/do/suggest?{query}'

98+ url = 'https://startpage.com/suggestions?{query}'

81−

82− resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')

99+ resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))

83− if len(resp) > 1:

100+ data = resp.json()

84− return resp

101+ return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

85− return []

86

102

87

103

88def swisscows(query, lang):

104def swisscows(query, lang):

@@ -120,7 +136,8 @@ def wikipedia(query, lang):

120 return []

136 return []

121

137

122

138

139+backends = {'brave': brave,

123−backends = {'dbpedia': dbpedia,

140+ 'dbpedia': dbpedia,

124 'duckduckgo': duckduckgo,

141 'duckduckgo': duckduckgo,

125 'google': google,

142 'google': google,

126 'startpage': startpage,

143 'startpage': startpage,

searx/data/__init__.py+39−14

@@ -1,25 +1,50 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+# lint: pylint

3+"""This module holds the *data* created by::

4+

5+ make data.all

6+

7+"""

8+

9+__all__ = [

10+ 'ENGINES_LANGUAGES',

11+ 'CURRENCIES',

12+ 'USER_AGENTS',

13+ 'EXTERNAL_URLS',

14+ 'WIKIDATA_UNITS',

15+ 'EXTERNAL_BANGS',

16+ 'OSM_KEYS_TAGS',

17+ 'ahmia_blacklist_loader',

18+]

19+

1import json

20import json

2from pathlib import Path

21from pathlib import Path

3

22

4−

5−__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'EXTERNAL_URLS', 'WIKIDATA_UNITS', 'EXTERNAL_BANGS',

6− 'bangs_loader', 'ahmia_blacklist_loader']

7data_dir = Path(__file__).parent

23data_dir = Path(__file__).parent

8

24

9

25

10−def load(filename):

26+def _load(filename):

11− with open(data_dir / filename, encoding='utf-8') as fd:

27+ with open(data_dir / filename, encoding='utf-8') as f:

12− return json.load(fd)

28+ return json.load(f)

13

29

14

30

15def ahmia_blacklist_loader():

31def ahmia_blacklist_loader():

32+ """Load data from `ahmia_blacklist.txt` and return a list of MD5 values of onion

33+ names. The MD5 values are fetched by::

34+

35+ searx_extra/update/update_ahmia_blacklist.py

36+

37+ This function is used by :py:mod:`searx.plugins.ahmia_filter`.

38+

39+ """

16− with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:

40+ with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as f:

17− return fd.read().split()

41+ return f.read().split()

18

42

19

43

20−ENGINES_LANGUAGES = load('engines_languages.json')

44+ENGINES_LANGUAGES = _load('engines_languages.json')

21−CURRENCIES = load('currencies.json')

45+CURRENCIES = _load('currencies.json')

22−USER_AGENTS = load('useragents.json')

46+USER_AGENTS = _load('useragents.json')

23−EXTERNAL_URLS = load('external_urls.json')

47+EXTERNAL_URLS = _load('external_urls.json')

24−WIKIDATA_UNITS = load('wikidata_units.json')

48+WIKIDATA_UNITS = _load('wikidata_units.json')

25−EXTERNAL_BANGS = load('external_bangs.json')

49+EXTERNAL_BANGS = _load('external_bangs.json')

50+OSM_KEYS_TAGS = _load('osm_keys_tags.json')

searx/data/ahmia_blacklist.txt+8485