b0888c6...master · searx

git.iko.soy/searx

46 files changed+15576−6302

since splitsnapshots

unifiedsplit

.github/workflows/data-update.yml+1−1 .github/workflows/integration.yml+32−3 .gitignore+2 .pylintrc+1−1 Dockerfile+5−4 README.rst+1−1 docs/dev/engine_overview.rst+1−1 manage+4−4 requirements-dev.txt+13−12 requirements.txt+7−7 searx/autocomplete.py+24−7 searx/data/ahmia_blacklist.txt+8485 searx/data/currencies.json+37−72 searx/data/engines_languages.json+45−2 searx/data/external_bangs.json+5826−5825 searx/data/useragents.json+4−3 searx/data/wikidata_units.json+33−5 searx/engines/__init__.py+1−1 searx/engines/command.py+1−1 searx/engines/gigablast.py+24−13 searx/engines/google.py+22−13 searx/engines/mysql_server.py+2 searx/engines/onesearch.py+57 searx/engines/prowlarr.py+80 searx/engines/qwant.py+175−69 searx/engines/startpage.py+100−20 searx/engines/tineye.py+86 searx/engines/xpath_flex.py+121 searx/engines/yahoo.py+93−91 searx/engines/yggtorrent.py+2−2 searx/network/__init__.py+28−29 searx/network/client.py+14−23 searx/network/network.py+71−16 searx/results.py+20−6 searx/search/__init__.py+5−2 searx/search/checker/impl.py+1−1 searx/search/processors/online.py+4−3 searx/settings.yml+69−18 searx/settings_loader.py+2−2 searx/templates/oscar/result_templates/products.html+2−1 searx/utils.py+12−6 searx/webapp.py+48−16 tests/unit/network/test_network.py+10−10 tests/unit/test_external_bangs.py−6 tests/unit/test_results.py+4−4 utils/searx.sh+1−1

.github/workflows/data-update.yml+1−1

@@ -29,7 +29,7 @@ jobs:

2929 - name: Set up Python

3030 uses: actions/setup-python@v2

3131 with:

32− python-version: '3.9'

32+ python-version: '3.10'

3333 architecture: 'x64'

3434

3535 - name: Install Python dependencies

.github/workflows/integration.yml+32−3

@@ -13,7 +13,7 @@ jobs:

1313 strategy:

1414 matrix:

1515 os: [ubuntu-20.04]

16− python-version: [3.6, 3.7, 3.8, 3.9]

16+ python-version: [3.7, 3.8, 3.9, "3.10"]

1717 steps:

1818 - name: Checkout

1919 uses: actions/checkout@v2

@@ -56,6 +56,17 @@ jobs:

5656 uses: actions/checkout@v2

5757 - name: Install Ubuntu packages

5858 run: sudo ./utils/searx.sh install packages

59+ - name: Set up Python

60+ uses: actions/setup-python@v2

61+ with:

62+ python-version: '3.9'

63+ architecture: 'x64'

64+ - name: Cache Python dependencies

65+ id: cache-python

66+ uses: actions/cache@v2

67+ with:

68+ path: ./local

69+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

5970 - name: Install node dependencies

6071 run: make V=1 node.env

6172 - name: Build themes

@@ -68,14 +79,21 @@ jobs:

6879 - name: Checkout

6980 uses: actions/checkout@v2

7081 with:

82+ fetch-depth: '0'

7183 persist-credentials: false

7284 - name: Install Ubuntu packages

7385 run: sudo ./utils/searx.sh install buildhost

7486 - name: Set up Python

7587 uses: actions/setup-python@v2

7688 with:

77− python-version: '3.9'

89+ python-version: '3.10'

7890 architecture: 'x64'

91+ - name: Cache Python dependencies

92+ id: cache-python

93+ uses: actions/cache@v2

94+ with:

95+ path: ./local

96+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

7997 - name: Build documentation

8098 run: |

8199 make V=1 docs.clean docs.html

@@ -97,7 +115,7 @@ jobs:

97115 - documentation

98116 env:

99117 DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

100− runs-on: ubuntu-18.04

118+ runs-on: ubuntu-20.04

101119 steps:

102120 - name: Checkout

103121 if: env.DOCKERHUB_USERNAME != null

@@ -105,6 +123,17 @@ jobs:

105123 with:

106124 # make sure "make docker.push" can get the git history

107125 fetch-depth: '0'

126+ - name: Set up Python

127+ uses: actions/setup-python@v2

128+ with:

129+ python-version: '3.9'

130+ architecture: 'x64'

131+ - name: Cache Python dependencies

132+ id: cache-python

133+ uses: actions/cache@v2

134+ with:

135+ path: ./local

136+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}

108137 - name: Set up QEMU

109138 if: env.DOCKERHUB_USERNAME != null

110139 uses: docker/setup-qemu-action@v1

.gitignore+2

@@ -26,3 +26,5 @@ dist/

2626local/

2727gh-pages/

2828searx.egg-info/

29+.env

30+geckodriver.log

.pylintrc+1−1

@@ -59,7 +59,7 @@ confidence=

5959# --enable=similarities". If you want to run only the classes checker, but have

6060# no Warning level messages displayed, use"--disable=all --enable=classes

6161# --disable=W"

62−disable=bad-whitespace, duplicate-code

62+disable=bad-whitespace, duplicate-code, consider-using-f-string

6363

6464# Enable the message, report, category or checker with the given id(s). You can

6565# either give multiple identifier separated by comma (,) or put this option

Dockerfile+5−4

@@ -1,4 +1,4 @@

1−FROM alpine:3.12

1+FROM alpine:3.15

22ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

33EXPOSE 8080

44VOLUME /etc/searx

@@ -46,18 +46,19 @@ RUN apk upgrade --no-cache \

4646 uwsgi \

4747 uwsgi-python3 \

4848 brotli \

49− && pip3 install --upgrade pip \

49+ && pip3 install --upgrade pip wheel setuptools \

5050 && pip3 install --no-cache -r requirements.txt \

5151 && apk del build-dependencies \

5252 && rm -rf /root/.cache

5353

54−COPY --chown=searx:searx . .

54+COPY searx ./searx

55+COPY dockerfiles ./dockerfiles

5556

5657ARG TIMESTAMP_SETTINGS=0

5758ARG TIMESTAMP_UWSGI=0

5859ARG VERSION_GITCOMMIT=unknown

5960

60−RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

61+RUN /usr/bin/python3 -m compileall -q searx; \

6162 touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \

6263 touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \

6364 if [ ! -z $VERSION_GITCOMMIT ]; then\

README.rst+1−1

@@ -61,7 +61,7 @@ our homepage_.

6161.. _homepage: https://searx.github.io/searx

6262

6363contact:

64− openhub_ // twitter_ // IRC: #searx @ freenode

64+ openhub_ // twitter_ // IRC: #searx @ Libera (irc.libera.chat)

6565

6666.. _openhub: https://www.openhub.net/p/searx

6767.. _twitter: https://twitter.com/Searx_engine

docs/dev/engine_overview.rst+1−1

@@ -171,7 +171,7 @@ headers set HTTP header information

171171data set HTTP data information

172172cookies set HTTP cookies

173173verify bool Performing SSL-Validity check

174−allow_redirects bool Follow redirects

174+follow_redirects bool Follow redirects

175175max_redirects int maximum redirects, hard limit

176176soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine

177177raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300

manage+4−4

@@ -13,7 +13,7 @@ source_dot_config

1313

1414PY_SETUP_EXTRAS='[test]'

1515NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"

16−GECKODRIVER_VERSION="v0.28.0"

16+GECKODRIVER_VERSION="v0.30.0"

1717# SPHINXOPTS=

1818

1919# These py files are linted by test.pylint(), all other files are linted by

@@ -224,7 +224,7 @@ docker.build() {

224224

225225 # build Docker image

226226 build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

227− sudo docker build \

227+ docker build \

228228 --build-arg GIT_URL="${GIT_URL}" \

229229 --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

230230 --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

@@ -236,8 +236,8 @@ docker.build() {

236236 -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

237237

238238 if [ "$1" = "push" ]; then

239− sudo docker push "${SEARX_IMAGE_NAME}:latest"

240− sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

239+ docker push "${SEARX_IMAGE_NAME}:latest"

240+ docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

241241 fi

242242 )

243243 dump_return $?

requirements-dev.txt+13−12

@@ -1,19 +1,20 @@

11mock==4.0.3

22nose2[coverage_plugin]==0.10.0

33cov-core==1.15.0

4−pycodestyle==2.7.0

5−pylint==2.10.2

6−splinter==0.15.0

7−transifex-client==0.14.3

8−selenium==3.141.0

9−twine==3.4.2

10−Pallets-Sphinx-Themes==2.0.1

4+pycodestyle==2.8.0

5+pylint==2.12.2

6+splinter==0.17.0

7+transifex-client==0.14.3; python_version < '3.10'

8+transifex-client==0.12.1; python_version == '3.10'

9+selenium==4.1.0

10+twine==3.7.1

11+Pallets-Sphinx-Themes==2.0.2

1112docutils==0.16

12−Sphinx==4.2.0

13−sphinx-issues==1.2.0

14−sphinx-jinja==1.1.1

13+Sphinx==4.4.0

14+sphinx-issues==3.0.1

15+sphinx-jinja==1.4.0

1516sphinx-tabs==3.2.0

1617sphinxcontrib-programoutput==0.17

1718sphinx-autobuild==2021.3.14

18−linuxdoc==20210324

19−aiounittest==1.4.0

19+linuxdoc==20211220

20+aiounittest==1.4.1

requirements.txt+7−7

@@ -1,16 +1,16 @@

1−certifi==2021.5.30

1+certifi==2021.10.8

22babel==2.9.1

33flask-babel==2.0.0

4−flask==1.1.2

5−jinja2==3.0.2

6−lxml==4.6.3

4+flask==2.0.2

5+jinja2==3.0.3

6+lxml==4.7.1

77pygments==2.8.0

88python-dateutil==2.8.2

9−pyyaml==5.4.1

10−httpx[http2]==0.19.0

9+pyyaml==6.0

10+httpx[http2]==0.21.3

1111Brotli==1.0.9

1212uvloop==0.16.0; python_version >= '3.7'

1313uvloop==0.14.0; python_version < '3.7'

14−httpx-socks[asyncio]==0.4.1

14+httpx-socks[asyncio]==0.7.2

1515langdetect==1.0.9

1616setproctitle==1.2.2

searx/autocomplete.py+24−7

@@ -24,6 +24,7 @@ from httpx import HTTPError

2424

2525

2626from searx import settings

27+from searx.data import ENGINES_LANGUAGES

2728from searx.network import get as http_get

2829from searx.exceptions import SearxEngineResponseException

2930

@@ -35,6 +36,22 @@ def get(*args, **kwargs):

3536 return http_get(*args, **kwargs)

3637

3738

39+def brave(query, lang):

40+ # brave search autocompleter

41+ url = 'https://search.brave.com/api/suggest?{query}'

42+

43+ resp = get(url.format(query=urlencode({'q': query})))

44+

45+ results = []

46+

47+ if resp.ok:

48+ data = loads(resp.text)

49+ for suggestion in data[1]:

50+ results.append(suggestion)

51+

52+ return results

53+

54+

3855def dbpedia(query, lang):

3956 # dbpedia autocompleter, no HTTPS

4057 autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

@@ -77,12 +94,11 @@ def google(query, lang):

7794

7895def startpage(query, lang):

7996 # startpage autocompleter

80− url = 'https://startpage.com/do/suggest?{query}'

81−

82− resp = get(url.format(query=urlencode({'query': query}))).text.split('\n')

83− if len(resp) > 1:

84− return resp

85− return []

97+ lui = ENGINES_LANGUAGES['startpage'].get(lang, 'english')

98+ url = 'https://startpage.com/suggestions?{query}'

99+ resp = get(url.format(query=urlencode({'q': query, 'segment': 'startpage.udog', 'lui': lui})))

100+ data = resp.json()

101+ return [e['text'] for e in data.get('suggestions', []) if 'text' in e]

86102

87103

88104def swisscows(query, lang):

@@ -120,7 +136,8 @@ def wikipedia(query, lang):

120136 return []

121137

122138

123−backends = {'dbpedia': dbpedia,

139+backends = {'brave': brave,

140+ 'dbpedia': dbpedia,

124141 'duckduckgo': duckduckgo,

125142 'google': google,

126143 'startpage': startpage,

searx/data/ahmia_blacklist.txt+8485