f7706a5 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

Merge pull request #2594 from return42/manage-script

f7706a5Adam Tauber6 years ago

.github/workflows/data-update.yml+1−2

compare with master

unifiedsplit

@@ -40,8 +40,7 @@ jobs:

40 env:

41 FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}

42 run: |

43− source local/py3/bin/activate

44− $FETCH_SCRIPT

43+ V=1 manage pyenv.cmd python "$FETCH_SCRIPT"

46 - name: Create Pull Request

45 - name: Create Pull Request

47 id: cpr

46 id: cpr

.github/workflows/integration.yml+4−3

@@ -59,7 +59,7 @@ jobs:

59 - name: Install node dependencies

60 run: make V=1 node.env

61 - name: Build themes

62− run: make V=1 themes

62+ run: make V=1 themes.all

64 documentation:

65 name: Documentation

@@ -77,14 +77,15 @@ jobs:

77 python-version: '3.9'

78 architecture: 'x64'

79 - name: Build documentation

80− run: SEARX_DEBUG=1 make V=1 ci-gh-pages

80+ run: |

81+ make V=1 docs.clean docs.html

81 - name: Deploy

82 - name: Deploy

82 if: github.ref == 'refs/heads/master'

83 if: github.ref == 'refs/heads/master'

83 uses: JamesIves/github-pages-deploy-action@3.7.1

84 uses: JamesIves/github-pages-deploy-action@3.7.1

84 with:

85 with:

85 GITHUB_TOKEN: ${{ github.token }}

86 GITHUB_TOKEN: ${{ github.token }}

86 BRANCH: gh-pages

87 BRANCH: gh-pages

87− FOLDER: build/gh-pages

88+ FOLDER: dist/docs

88 CLEAN: true # Automatically remove deleted files from the deploy branch

89 CLEAN: true # Automatically remove deleted files from the deploy branch

90 dockers:

91 dockers:

Makefile+78−237

@@ -1,266 +1,107 @@

1# -*- coding: utf-8; mode: makefile-gmake -*-

2+# SPDX-License-Identifier: AGPL-3.0-or-later

2.DEFAULT_GOAL=help

4.DEFAULT_GOAL=help

5+export MTOOLS=./manage

4include utils/makefile.include

7include utils/makefile.include

6−PYOBJECTS = searx

7−DOC = docs

8−PY_SETUP_EXTRAS ?= \[test\]

9−PYLINT_SEARX_DISABLE_OPTION := I,C,R,W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,E1136

10−PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES := supported_languages,language_aliases

11−

12−include utils/makefile.python

13−include utils/makefile.sphinx

14−

15all: clean install

9all: clean install

17−PHONY += help-min help-all help

11+PHONY += help

18−

19−help: help-min

20− @echo ''

21− @echo 'to get more help: make help-all'

22−

23−help-min:

24− @echo ' test - run developer tests'

25− @echo ' docs - build documentation'

26− @echo ' docs-live - autobuild HTML documentation while editing'

27− @echo ' run - run developer instance'

28− @echo ' install - developer install (./local)'

29− @echo ' uninstall - uninstall (./local)'

30− @echo ' gh-pages - build docs & deploy on gh-pages branch'

31− @echo ' clean - drop builds and environments'

32− @echo ' project - re-build generic files of the searx project'

33− @echo ' buildenv - re-build environment files (aka brand)'

34− @echo ' themes - re-build build the source of the themes'

35− @echo ' docker - build Docker image'

36− @echo ' node.env - download & install npm dependencies locally'

37− @echo ''

38− @$(MAKE) -e -s make-help

39−

40−help-all: help-min

41− @echo ''

42− @$(MAKE) -e -s python-help

43− @echo ''

44− @$(MAKE) -e -s docs-help

46−PHONY += install

13+help:

47−install: buildenv pyenvinstall

14+ @./manage --help

15+ @echo '----'

16+ @echo 'run - run developer instance'

17+ @echo 'install - developer install of searx into virtualenv'

18+ @echo 'uninstall - uninstall developer installation'

19+ @echo 'clean - clean up working tree'

20+ @echo 'search.checker - check search engines'

21+ @echo 'test - run shell & CI tests'

22+ @echo 'test.sh - test shell scripts'

23+ @echo 'ci.test - run CI tests'

49−PHONY += uninstall

50−uninstall: pyenvuninstall

51−

52−PHONY += clean

53−clean: pyclean docs-clean node.clean test.clean

54− $(call cmd,common_clean)

56PHONY += run

26PHONY += run

57−run: buildenv pyenvinstall

27+run: install

58 $(Q) ( \

28 $(Q) ( \

59 sleep 2 ; \

29 sleep 2 ; \

60 xdg-open http://127.0.0.1:8888/ ; \

30 xdg-open http://127.0.0.1:8888/ ; \

61 ) &

31 ) &

62− SEARX_DEBUG=1 $(PY_ENV)/bin/python ./searx/webapp.py

32+ SEARX_DEBUG=1 ./manage pyenv.cmd python ./searx/webapp.py

63−

64−# docs

65−# ----

66−

67−sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes

68−

69−PHONY += docs

70−docs: sphinx-doc-prebuilds

71− $(call cmd,sphinx,html,docs,docs)

72−

73−PHONY += docs-live

74−docs-live: sphinx-doc-prebuilds

75− $(call cmd,sphinx_autobuild,html,docs,docs)

76−

77−PHONY += prebuild-includes

78−prebuild-includes:

79− $(Q)mkdir -p $(DOCS_BUILD)/includes

80− $(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst

81− $(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst

82− $(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst

83−

84−

85−$(GH_PAGES)::

86− @echo "doc available at --> $(DOCS_URL)"

87−

88−# update project files

89−# --------------------

90−

91−PHONY += project engines.languages useragents.update buildenv

92−

93−project: buildenv useragents.update engines.languages

94−

95−engines.languages: pyenvinstall

96− $(Q)echo "fetch languages .."

97− $(Q)$(PY_ENV_ACT); python ./searx_extra/update/update_languages.py

98− $(Q)echo "updated searx/data/engines_languages.json"

99− $(Q)echo "updated searx/languages.py"

100−

101−useragents.update: pyenvinstall

102− $(Q)echo "fetch useragents .."

103− $(Q)$(PY_ENV_ACT); python ./searx_extra/update/update_firefox_version.py

104− $(Q)echo "updated searx/data/useragents.json with the most recent versions of Firefox."

105−

106−buildenv: pyenv

107− $(Q)$(PY_ENV_ACT); SEARX_DEBUG=1 python utils/build_env.py

108−

109−# node / npm

110−# ----------

111−

112−node.env: buildenv

113− $(Q)./manage.sh npm_packages

114

115−node.clean:

34+PHONY += install uninstall

116− $(Q)echo "CLEAN locally installed npm dependencies"

35+install uninstall:

117− $(Q)rm -rf \

36+ $(Q)./manage pyenv.$@

118− ./node_modules \

119− ./package-lock.json \

120− ./searx/static/themes/oscar/package-lock.json \

121− ./searx/static/themes/oscar/node_modules \

122− ./searx/static/themes/simple/package-lock.json \

123− ./searx/static/themes/simple/node_modules

124

125−# build themes

38+PHONY += clean

126−# ------------

39+clean: py.clean docs.clean node.clean test.clean

127−

40+ $(Q)./manage build_msg CLEAN "common files"

128−PHONY += themes themes.oscar themes.simple

41+ $(Q)find . -name '*.orig' -exec rm -f {} +

129−themes: buildenv themes.oscar themes.simple

42+ $(Q)find . -name '*.rej' -exec rm -f {} +

130−

43+ $(Q)find . -name '*~' -exec rm -f {} +

131−quiet_cmd_lessc = LESSC $3

44+ $(Q)find . -name '*.bak' -exec rm -f {} +

132− cmd_lessc = PATH="$$(npm bin):$$PATH" \

45+

133− lessc --clean-css="--s1 --advanced --compatibility=ie9" "searx/static/$2" "searx/static/$3"

134−

135−quiet_cmd_grunt = GRUNT $2

136− cmd_grunt = PATH="$$(npm bin):$$PATH" \

137− grunt --gruntfile "$2"

138−

139−themes.oscar: node.env

140− $(Q)echo '[!] build oscar theme'

141− $(call cmd,grunt,searx/static/themes/oscar/gruntfile.js)

142−

143−themes.simple: node.env

144− $(Q)echo '[!] build simple theme'

145− $(call cmd,grunt,searx/static/themes/simple/gruntfile.js)

146−

147−

148−# docker

149−# ------

150−

151−PHONY += docker

152−docker: buildenv

153− $(Q)./manage.sh docker_build

154−

155−docker.push: buildenv

156− $(Q)./manage.sh docker_build push

157−

158−# gecko

159−# -----

160−

161−PHONY += gecko.driver

162−gecko.driver:

163− $(PY_ENV_ACT); ./manage.sh install_geckodriver

164−

165−# search.checker

46+PHONY += search.checker search.checker.%

166−# --------------

167−

168−search.checker: pyenvinstall

47+search.checker: install

169− $(Q)$(PY_ENV_ACT); searx-checker -v

48+ $(Q)./manage pyenv.cmd searx-checker -v

170−

49+

171−ENGINE_TARGETS=$(patsubst searx/engines/%.py,search.checker.%,$(wildcard searx/engines/[!_]*.py))

50+search.checker.%: install

172−

173−$(ENGINE_TARGETS): pyenvinstall

174− $(Q)$(PY_ENV_ACT); searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"

51+ $(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"

175−

52+

176−

53+PHONY += ci.test test test.sh

177−# test

178−# ----

179−

180−PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot

54+ci.test: test.pep8 test.pylint test.unit test.robot

181−test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot

55+test: ci.test

182−

56+test.sh:

183−PYLINT_FILES=\

57+ $(Q)shellcheck -x -s bash \

184− searx/preferences.py \

58+ utils/brand.env \

185− searx/testing.py \

59+ ./manage \

186− searx/engines/gigablast.py \

60+ utils/lib.sh \

187− searx/engines/deviantart.py \

61+ utils/filtron.sh \

188− searx/engines/digg.py \

62+ utils/searx.sh \

189− searx/engines/google.py \

63+ utils/morty.sh \

190− searx/engines/google_news.py \

64+ utils/lxc.sh \

191− searx/engines/google_videos.py \

65+ utils/lxc-searx.env \

192− searx/engines/google_images.py \

66+ .config.sh

193− searx/engines/mediathekviewweb.py \

67+ $(Q)./manage build_msg TEST "$@ OK"

194− searx/engines/meilisearch.py \

195− searx/engines/solidtorrents.py \

196− searx/engines/solr.py \

197− searx/engines/google_scholar.py \

198− searx/engines/yahoo_news.py \

199− searx/engines/apkmirror.py \

200− searx_extra/update/update_external_bangs.py

201−

202−test.pylint: pyenvinstall

203− $(call cmd,pylint,$(PYLINT_FILES))

204− $(call cmd,pylint,\

205− --disable=$(PYLINT_SEARX_DISABLE_OPTION) \

206− --additional-builtins=$(PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES) \

207− searx/engines \

208− )

209− $(call cmd,pylint,\

210− --disable=$(PYLINT_SEARX_DISABLE_OPTION) \

211− --ignore=searx/engines \

212− searx tests \

213− )

214

215−# ignored rules:

216−# E402 module level import not at top of file

217−# W503 line break before binary operator

218

219−# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported!

70+# wrap ./manage script

220−test.sh:

221− shellcheck -x -s bash utils/brand.env

222− shellcheck -x utils/lib.sh

223− shellcheck -x utils/filtron.sh

224− shellcheck -x utils/searx.sh

225− shellcheck -x utils/morty.sh

226− shellcheck -x utils/lxc.sh

227− shellcheck -x utils/lxc-searx.env

228− shellcheck -x .config.sh

229

230−test.pep8: pyenvinstall

72+MANAGE += buildenv

231− @echo "TEST pycodestyle (formerly pep8)"

73+MANAGE += babel.compile

232− $(Q)$(PY_ENV_ACT); pycodestyle --exclude='searx/static, searx/languages.py, $(foreach f,$(PYLINT_FILES),$(f),)' \

74+MANAGE += data.all data.languages data.useragents

233− --max-line-length=120 --ignore "E117,E252,E402,E722,E741,W503,W504,W605" searx tests

75+MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean

76+MANAGE += docker.build docker.push

77+MANAGE += gecko.driver

78+MANAGE += node.env node.clean

79+MANAGE += py.build py.clean

80+MANAGE += pyenv pyenv.install pyenv.uninstall

81+MANAGE += pypi.upload pypi.upload.test

82+MANAGE += test.pylint test.pep8 test.unit test.coverage test.robot test.clean

83+MANAGE += themes.all themes.oscar themes.simple themes.bootstrap

234

235−test.unit: pyenvinstall

85+PHONY += $(MANAGE)

236− @echo "TEST tests/unit"

237− $(Q)$(PY_ENV_ACT); python -m nose2 -s tests/unit

238

239−test.coverage: pyenvinstall

87+$(MANAGE):

240− @echo "TEST unit test coverage"

88+ $(Q)$(MTOOLS) $@

241− $(Q)$(PY_ENV_ACT); \

242− python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit \

243− && coverage report \

244− && coverage html \

245

246−test.robot: pyenvinstall gecko.driver

90+# deprecated

247− @echo "TEST robot"

248− $(Q)$(PY_ENV_ACT); PYTHONPATH=. python searx/testing.py robot

249

250−test.clean:

92+PHONY += docs docs-clean docs-live docker themes

251− @echo "CLEAN intermediate test stuff"

252− $(Q)rm -rf geckodriver.log .coverage coverage/

253

94+docs: docs.html

95+ $(Q)./manage build_msg WARN $@ is deprecated use docs.html

254

255−# travis

97+docs-clean: docs.clean

256−# ------

98+ $(Q)./manage build_msg WARN $@ is deprecated use docs.clean

257

258−PHONY += ci.test

100+docs-live: docs.live

259−ci.test:

101+ $(Q)./manage build_msg WARN $@ is deprecated use docs.live

260− $(PY_ENV_BIN)/python -c "import yaml" || make clean

261− $(MAKE) test

262

102

263−travis.codecov:

103+docker: docker.build

264− $(Q)$(PY_ENV_BIN)/python -m pip install codecov

104+ $(Q)./manage build_msg WARN $@ is deprecated use docker.build

265

105

266−.PHONY: $(PHONY)

106+themes: themes.all

107+ $(Q)./manage build_msg WARN $@ is deprecated use themes.all

docs/admin/buildhosts.rst+4−4

@@ -49,9 +49,9 @@ Build docs

49 - dvisvgm_

51Most of the sphinx requirements are installed from :origin:`setup.py` and the

52−docs can be build from scratch with ``make docs``. For better math and image

52+docs can be build from scratch with ``make docs.html``. For better math and

53−processing additional packages are needed. The XeTeX_ needed not only for PDF

53+image processing additional packages are needed. The XeTeX_ needed not only for

54−creation, its also needed for :ref:`math` when HTML output is build.

54+PDF creation, its also needed for :ref:`math` when HTML output is build.

56To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered

57as images (``sphinx.ext.imgmath`` extension).

@@ -64,7 +64,7 @@ to ``imgmath``:

64 :start-after: # sphinx.ext.imgmath setup

65 :end-before: # sphinx.ext.imgmath setup END

67−If your docs build (``make docs``) shows warnings like this::

67+If your docs build (``make docs.html``) shows warnings like this::

69 WARNING: dot(1) not found, for better output quality install \

70 graphviz from https://www.graphviz.org

docs/admin/installation-docker.rst+1−1

@@ -51,7 +51,7 @@ It's also possible to build searx from the embedded Dockerfile.

52 git clone https://github.com/searx/searx.git

53 cd searx

54− make docker

54+ make docker.build

57Public instance

docs/blog/lxcdev-202006.rst+3−3

@@ -31,7 +31,7 @@ might fail in some aspects we should not overlook.

32 The environment in which we run all our development processes matters!

34−The :ref:`makefile` and the :ref:`make pyenv` encapsulate a lot for us, but they

34+The :ref:`makefile` and the :ref:`make install` encapsulate a lot for us, but they

35do not have access to all prerequisites. For example, there may have

36dependencies on packages that are installed on the developer's desktop, but

37usually are not preinstalled on a server or client system. Another examples

@@ -356,7 +356,7 @@ daily usage:

356 .. code:: sh

357

358 $ sudo -H ./utils/lxc.sh cmd searx-archlinux \

359− make docs

359+ make docs.html

360

361.. _blog-lxcdev-202006 abstract:

362

@@ -407,7 +407,7 @@ To get remarks from the suite of the archlinux container we can use:

407 ...

408 [searx-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx

409 [searx-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/

410− [searx-archlinux] INFO: (eth0) docs-live: http://10.174.184.156:8080/

410+ [searx-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/

411 [searx-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65]

412 ...

413

docs/conf.py+1−7

@@ -1,7 +1,7 @@

1# -*- coding: utf-8 -*-

2+# SPDX-License-Identifier: AGPL-3.0-or-later

3import sys, os

4import sys, os

4−from sphinx_build_tools import load_sphinx_config

5from pallets_sphinx_themes import ProjectLink

7from searx import brand

@@ -128,9 +128,3 @@ html_show_sourcelink = False

128latex_documents = [

129 (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual")

130]

131−

132−# ------------------------------------------------------------------------------

133−# Since loadConfig overwrites settings from the global namespace, it has to be

134−# the last statement in the conf.py file

135−# ------------------------------------------------------------------------------

136−load_sphinx_config(globals())

docs/dev/contribution_guide.rst+11−22

@@ -132,11 +132,11 @@ Here is an example which makes a complete rebuild:

132

133.. code:: sh

134

135− $ make docs-clean docs

135+ $ make docs.clean docs.html

136 ...

137 The HTML pages are in dist/docs.

138

139−.. _make docs-live:

139+.. _make docs.live:

140

141live build

142----------

@@ -144,19 +144,19 @@ live build

144.. _sphinx-autobuild:

145 https://github.com/executablebooks/sphinx-autobuild/blob/master/README.md

146

147−.. sidebar:: docs-clean

147+.. sidebar:: docs.clean

148

149 It is recommended to assert a complete rebuild before deploying (use

150− ``docs-clean``).

150+ ``docs.clean``).

151

152Live build is like WYSIWYG. If you want to edit the documentation, its

153−recommended to use. The Makefile target ``docs-live`` builds the docs, opens

153+recommended to use. The Makefile target ``docs.live`` builds the docs, opens

154URL in your favorite browser and rebuilds every time a reST file has been

155changed.

156

157.. code:: sh

158

159− $ make docs-live

159+ $ make docs.live

160 ...

161 The HTML pages are in dist/docs.

162 ... Serving on http://0.0.0.0:8000

@@ -169,7 +169,7 @@ argument. E.g to find and use a free port, use:

169

170.. code:: sh

171

172− $ SPHINXOPTS="--port 0" make docs-live

172+ $ SPHINXOPTS="--port 0" make docs.live

173 ...

174 ... Serving on http://0.0.0.0:50593

175 ...

@@ -180,21 +180,10 @@ argument. E.g to find and use a free port, use:

180deploy on github.io

181-------------------

182

183−To deploy documentation at :docs:`github.io <.>` use Makefile target

183+To deploy documentation at :docs:`github.io <.>` use Makefile target :ref:`make

184−:ref:`make gh-pages`, which will builds the documentation, clones searx into a sub

184+docs.gh-pages`, which builds the documentation and runs all the needed git add,

185−folder ``gh-pages``, cleans it, copies the doc build into and runs all the

186−needed git add, commit and push:

185+commit and push:

187

186

188.. code:: sh

187.. code:: sh

189

188

190− $ make docs-clean gh-pages

189+ $ make docs.clean docs.gh-pages

191− ...

192− SPHINX docs --> file://<...>/dist/docs

193− The HTML pages are in dist/docs.

194− ...

195− Cloning into 'gh-pages' ...

196− ...

197− cd gh-pages; git checkout gh-pages >/dev/null

198− Switched to a new branch 'gh-pages'

199− ...

200− doc available at --> https://searx.github.io/searx

docs/dev/makefile.rst+116−112

@@ -1,33 +1,33 @@

1.. _makefile:

3−================

3+========

4−Makefile Targets

4+Makefile

5−================

5+========

7.. _gnu-make: https://www.gnu.org/software/make/manual/make.html#Introduction

9.. sidebar:: build environment

11− Before looking deeper at the targets, first read about :ref:`make pyenv`.

11+ Before looking deeper at the targets, first read about :ref:`make

12+ install`.

13 To install system requirements follow :ref:`buildhosts`.

14 To install system requirements follow :ref:`buildhosts`.

15−With the aim to simplify development cycles, started with :pull:`1756` a

16+All relevant build tasks are implemented in :origin:`manage.sh` and for CI or

16−``Makefile`` based boilerplate was added. If you are not familiar with

17+IDE integration a small ``Makefile`` wrapper is available. If you are not

17−Makefiles, we recommend to read gnu-make_ introduction.

18+familiar with Makefiles, we recommend to read gnu-make_ introduction.

19The usage is simple, just type ``make {target-name}`` to *build* a target.

20The usage is simple, just type ``make {target-name}`` to *build* a target.

20Calling the ``help`` target gives a first overview (``make help``):

21Calling the ``help`` target gives a first overview (``make help``):

22.. program-output:: bash -c "cd ..; make --no-print-directory help"

23.. program-output:: bash -c "cd ..; make --no-print-directory help"

24−

25.. contents:: Contents

26 :depth: 2

27 :local:

28 :backlinks: entry

30−.. _make pyenv:

30+.. _make install:

32Python environment

33==================

@@ -36,31 +36,42 @@ Python environment

37 ``source ./local/py3/bin/activate``

39−With Makefile we do no longer need to build up the virtualenv manually (as

39+We do no longer need to build up the virtualenv manually. Jump into your git

40−described in the :ref:`devquickstart` guide). Jump into your git working tree

41−and release a ``make pyenv``:

40+working tree and release a ``make install`` to get a virtualenv with a

42−

41+*developer install* of searx (:origin:`setup.py`). ::

43−.. code:: sh

45 $ cd ~/searx-clone

43 $ cd ~/searx-clone

46− $ make pyenv

44+ $ make install

47− PYENV usage: source ./local/py3/bin/activate

45+ PYENV [virtualenv] installing ./requirements*.txt into local/py3

48 ...

46 ...

47+ PYENV OK

48+ PYENV [install] pip install -e 'searx[test]'

49+ ...

50+ Successfully installed argparse-1.4.0 searx

51+ BUILDENV INFO:searx:load the default settings from ./searx/settings.yml

52+ BUILDENV INFO:searx:Initialisation done

53+ BUILDENV build utils/brand.env

50−With target ``pyenv`` a development environment (aka virtualenv) was build up in

55+If you release ``make install`` multiple times the installation will only

51−``./local/py3/``. To make a *developer install* of searx (:origin:`setup.py`)

56+rebuild if the sha256 sum of the *requirement files* fails. With other words:

52−into this environment, use make target ``install``:

57+the check fails if you edit the requirements listed in

53−

58+:origin:`requirements-dev.txt` and :origin:`requirements.txt`). ::

54−.. code:: sh

56 $ make install

60 $ make install

57− PYENV usage: source ./local/py3/bin/activate

61+ PYENV OK

58− PYENV using virtualenv from ./local/py3

62+ PYENV [virtualenv] requirements.sha256 failed

59− PYENV install .

63+ [virtualenv] - 6cea6eb6def9e14a18bf32f8a3e... ./requirements-dev.txt

60−

64+ [virtualenv] - 471efef6c73558e391c3adb35f4... ./requirements.txt

61−You have never to think about intermediate targets like ``pyenv`` or

65+ ...

62−``install``, the ``Makefile`` chains them as requisites. Just run your main

66+ PYENV [virtualenv] installing ./requirements*.txt into local/py3

63−target.

67+ ...

68+ PYENV OK

69+ PYENV [install] pip install -e 'searx[test]'

70+ ...

71+ Successfully installed argparse-1.4.0 searx

72+ BUILDENV INFO:searx:load the default settings from ./searx/settings.yml

73+ BUILDENV INFO:searx:Initialisation done

74+ BUILDENV build utils/brand.env

65.. sidebar:: drop environment

76.. sidebar:: drop environment

@@ -68,10 +79,7 @@ target.

68 <make clean>` first.

79 <make clean>` first.

70If you think, something goes wrong with your ./local environment or you change

81If you think, something goes wrong with your ./local environment or you change

71−the :origin:`setup.py` file (or the requirements listed in

82+the :origin:`setup.py` file, you have to call :ref:`make clean`.

72−:origin:`requirements-dev.txt` and :origin:`requirements.txt`), you have to call

73−:ref:`make clean`.

74−

76.. _make run:

84.. _make run:

@@ -81,117 +89,113 @@ the :origin:`setup.py` file (or the requirements listed in

81To get up a running a developer instance simply call ``make run``. This enables

89To get up a running a developer instance simply call ``make run``. This enables

82*debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py``

90*debug* option in :origin:`searx/settings.yml`, starts a ``./searx/webapp.py``

83instance, disables *debug* option again and opens the URL in your favorite WEB

91instance, disables *debug* option again and opens the URL in your favorite WEB

84−browser (:man:`xdg-open`):

92+browser (:man:`xdg-open`)::

85−

86−.. code:: sh

88− $ make run

94+ $ make run

89− PYENV usage: source ./local/py3/bin/activate

95+ PYENV OK

90− PYENV install .

91− ./local/py3/bin/python ./searx/webapp.py

96+ SEARX_DEBUG=1 ./manage.sh pyenv.cmd python ./searx/webapp.py

92− ...

97+ ...

93− INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)

98+ INFO:werkzeug: * Running on http://127.0.0.1:8888/ (Press CTRL+C to quit)

94− ...

96.. _make clean:

100.. _make clean:

101

98``make clean``

102``make clean``

99==============

103==============

100

104

101−Drop all intermediate files, all builds, but keep sources untouched. Includes

105+Drop all intermediate files, all builds, but keep sources untouched. Before

102−target ``pyclean`` which drops ./local environment. Before calling ``make

103−clean`` stop all processes using :ref:`make pyenv`.

106+calling ``make clean`` stop all processes using :ref:`make install`. ::

104−

105−.. code:: sh

106

107

107 $ make clean

108 $ make clean

108− CLEAN pyclean

109+ CLEAN pyenv

109− CLEAN clean

110+ PYENV [virtualenv] drop ./local/py3

111+ CLEAN docs -- ./build/docs ./dist/docs

112+ CLEAN locally installed npm dependencies

113+ CLEAN test stuff

114+ CLEAN common files

110

115

111.. _make docs:

116.. _make docs:

112

117

113−``make docs docs-live docs-clean``

118+``make docs docs.autobuild docs.clean``

114−==================================

119+=======================================

115

120

116−We describe the usage of the ``doc*`` targets in the :ref:`How to contribute /

121+We describe the usage of the ``doc.*`` targets in the :ref:`How to contribute /

117Documentation <contrib docs>` section. If you want to edit the documentation

122Documentation <contrib docs>` section. If you want to edit the documentation

118−read our :ref:`make docs-live` section. If you are working in your own brand,

123+read our :ref:`make docs.live` section. If you are working in your own brand,

119adjust your :ref:`settings global`.

124adjust your :ref:`settings global`.

120

125

121−.. _make books:

126+.. _make docs.gh-pages:

122

127

123−``make books/{name}.html books/{name}.pdf``

128+``make docs.gh-pages``

124−===========================================

129+======================

125

130

126−.. _intersphinx: https://www.sphinx-doc.org/en/stable/ext/intersphinx.html

131+To deploy on github.io first adjust your :ref:`settings global`. For any

127−.. _XeTeX: https://tug.org/xetex/

132+further read :ref:`deploy on github.io`.

128

133

129−.. sidebar:: info

134+.. _make test:

130

135

131− To build PDF a XeTeX_ is needed, see :ref:`buildhosts`.

136+``make test``

137+=============

132

138

139+Runs a series of tests: :ref:`make test.pylint`, ``test.pep8``, ``test.unit``

140+and ``test.robot``. You can run tests selective, e.g.::

133

141

134−The ``books/{name}.*`` targets are building *books*. A *book* is a

142+ $ make test.pep8 test.unit test.sh

135−sub-directory containing a ``conf.py`` file. One example is the user handbook

143+ TEST test.pep8 OK

136−which can deployed separately (:origin:`docs/user/conf.py`). Such ``conf.py``

144+ ...

137−do inherit from :origin:`docs/conf.py` and overwrite values to fit *book's*

145+ TEST test.unit OK

138−needs.

146+ ...

147+ TEST test.sh OK

139

148

140−With the help of Intersphinx_ (:ref:`reST smart ref`) the links to searx’s

149+.. _make test.sh:

141−documentation outside of the book will be bound by the object inventory of

142−``DOCS_URL``. Take into account that URLs will be picked from the inventary at

143−documentation's build time.

144

150

145−Use ``make docs-help`` to see which books available:

151+``make test.sh``

152+================

146

153

147−.. program-output:: bash -c "cd ..; make --no-print-directory docs-help"

154+:ref:`sh lint` / if you have changed some bash scripting run this test before

148− :ellipsis: 0,-6

155+commit.

149

156

157+.. _make test.pylint:

150

158

151−.. _make gh-pages:

159+``make test.pylint``

160+====================

152

161

153−``make gh-pages``

162+.. _Pylint: https://www.pylint.org/

154−=================

155

163

156−To deploy on github.io first adjust your :ref:`settings global`. For any

164+Pylint_ is known as one of the best source-code, bug and quality checker for the

157−further read :ref:`deploy on github.io`.

165+Python programming language. The pylint profile we use at searx project is

166+found in project's root folder :origin:`.pylintrc`.

158

167

159−.. _make test:

168+.. _make search.checker:

160

169

161−``make test``

170+``search.checker.{engine name}``

162−=============

171+================================

163

172

164−Runs a series of tests: ``test.pep8``, ``test.unit``, ``test.robot`` and does

173+To check all engines::

165−additional :ref:`pylint checks <make pylint>`. You can run tests selective,

166−e.g.:

167

174

168−.. code:: sh

175+ make search.checker

169

176

170− $ make test.pep8 test.unit test.sh

177+To check a engine with whitespace in the name like *google news* replace space

171− . ./local/py3/bin/activate; ./manage.sh pep8_check

178+by underline::

172− [!] Running pep8 check

173− . ./local/py3/bin/activate; ./manage.sh unit_tests

174− [!] Running unit tests

175

179

176−.. _make pylint:

180+ make search.checker.google_news

177

181

178−``make pylint``

182+To see HTTP requests and more use SEARX_DEBUG::

179−===============

180

183

181−.. _Pylint: https://www.pylint.org/

184+ make SEARX_DEBUG=1 search.checker.google_news

185+

186+.. _3xx: https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_redirection

182

187

183−Before commiting its recommend to do some (more) linting. Pylint_ is known as

188+To filter out HTTP redirects (3xx_)::

184−one of the best source-code, bug and quality checker for the Python programming

185−language. Pylint_ is not yet a quality gate within our searx project (like

186−:ref:`test.pep8 <make test>` it is), but Pylint_ can help to improve code

187−quality anyway. The pylint profile we use at searx project is found in

188−project's root folder :origin:`.pylintrc`.

189

190−Code quality is a ongoing process. Don't try to fix all messages from Pylint,

190+ make SEARX_DEBUG=1 search.checker.google_news | grep -A1 "HTTP/1.1\" 3[0-9][0-9]"

191−run Pylint and check if your changed lines are bringing up new messages. If so,

191+ ...

192−fix it. By this, code quality gets incremental better and if there comes the

192+ Engine google news Checking

193−day, the linting is balanced out, we might decide to add Pylint as a quality

193+ https://news.google.com:443 "GET /search?q=life&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0

194−gate.

194+ https://news.google.com:443 "GET /search?q=life&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None

195+ --

196+ https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0

197+ https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None

198+ --

195

199

196

200

197``make pybuild``

201``make pybuild``

@@ -200,9 +204,7 @@ gate.

200.. _PyPi: https://pypi.org/

204.. _PyPi: https://pypi.org/

201.. _twine: https://twine.readthedocs.io/en/latest/

205.. _twine: https://twine.readthedocs.io/en/latest/

202

206

203−Build Python packages in ``./dist/py``.

207+Build Python packages in ``./dist/py``::

204−

205−.. code:: sh

206

208

207 $ make pybuild

209 $ make pybuild

208 ...

210 ...

@@ -210,9 +212,11 @@ Build Python packages in ``./dist/py``.

210 running sdist

212 running sdist

211 running egg_info

213 running egg_info

212 ...

214 ...

215+ running bdist_wheel

216+

213− $ ls ./dist/py/

217+ $ ls ./dist

214− searx-0.15.0-py3-none-any.whl searx-0.15.0.tar.gz

218+ searx-0.18.0-py3-none-any.whl searx-0.18.0.tar.gz

215

219

216−To upload packages to PyPi_, there is also a ``upload-pypi`` target. It needs

220+To upload packages to PyPi_, there is also a ``pypi.upload`` target (to test use

217−twine_ to be installed. Since you are not the owner of :pypi:`searx` you will

221+``pypi.upload.test``). Since you are not the owner of :pypi:`searx` you will

218−never need the latter.

222+never need to upload.

docs/dev/reST.rst+6−7

@@ -15,8 +15,8 @@ generated and deployed at :docs:`github.io <.>`. For build prerequisites read

15:ref:`docs build`.

17The source files of Searx's documentation are located at :origin:`docs`. Sphinx

18−assumes source files to be encoded in UTF-8 by defaul. Run :ref:`make docs-live

18+assumes source files to be encoded in UTF-8 by defaul. Run :ref:`make docs.live

19−<make docs-live>` to build HTML while editing.

19+<make docs.live>` to build HTML while editing.

21.. sidebar:: Further reading

@@ -1276,13 +1276,12 @@ Templating

1276

1277.. sidebar:: Build environment

1278

1279− All *generic-doc* tasks are running in the :ref:`build environment <make

1279+ All *generic-doc* tasks are running in the :ref:`make install`.

1280− pyenv>`.

1281

1280

1282Templating is suitable for documentation which is created generic at the build

1281Templating is suitable for documentation which is created generic at the build

1283−time. The sphinx-jinja_ extension evaluates jinja_ templates in the :ref:`build

1282+time. The sphinx-jinja_ extension evaluates jinja_ templates in the :ref:`make

1284−environment <make pyenv>` (with searx modules installed). We use this e.g. to

1283+install` (with searx modules installed). We use this e.g. to build chapter:

1285−build chapter: :ref:`engines generic`. Below the jinja directive from the

1284+:ref:`engines generic`. Below the jinja directive from the

1286:origin:`docs/admin/engines.rst` is shown:

1285:origin:`docs/admin/engines.rst` is shown:

1287

1286

1288.. literalinclude:: ../admin/engines.rst

1287.. literalinclude:: ../admin/engines.rst

docs/user/conf.pydeleted−21

@@ -1,21 +0,0 @@

1−# -*- coding: utf-8; mode: python -*-

2−"""Configuration for the Searx user handbook

3−"""

4−project = 'Searx User-HB'

5−version = release = VERSION_STRING

6−

7−intersphinx_mapping['searx'] = (brand.DOCS_URL, None)

8−

9−# Grouping the document tree into LaTeX files. List of tuples

10−# (source start file, target name, title,

11−# author, documentclass [howto, manual, or own class]).

12−latex_documents = [

13− ('index' # startdocname

14− , 'searx-user-hb.tex' # targetname

15− , '' # take title from .rst

16− , author # author

17− , 'howto' # documentclass

18− , False # toctree_only

19− ),

20−]

21−

docs/utils/lxc.sh.rst+3−3

@@ -119,15 +119,15 @@ of coffee).::

119

120To build (live) documentation inside a archlinux_ container::

121

122− sudo -H ./utils/lxc.sh cmd searx-archlinux make docs-clean docs-live

122+ sudo -H ./utils/lxc.sh cmd searx-archlinux make docs.clean docs.live

123 ...

124 [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080

125

126To get IP of the container and the port number *live docs* is listening::

127

128− $ sudo ./utils/lxc.sh show suite | grep docs-live

128+ $ sudo ./utils/lxc.sh show suite | grep docs.live

129 ...

130− [searx-archlinux] INFO: (eth0) docs-live: http://n.n.n.12:8080/

130+ [searx-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/

131

132

133.. _lxc.sh help:

managenew file+501

@@ -0,0 +1,501 @@

1+#!/usr/bin/env bash

2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

3+# SPDX-License-Identifier: AGPL-3.0-or-later

4+# shellcheck disable=SC2031

6+# shellcheck source=utils/lib.sh

7+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"

8+# shellcheck source=utils/brand.env

9+source "${REPO_ROOT}/utils/brand.env"

10+source_dot_config

11+

12+# config

13+

14+PY_SETUP_EXTRAS='[test]'

15+NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"

16+GECKODRIVER_VERSION="v0.28.0"

17+# SPHINXOPTS=

18+

19+# These py files are linted by test.pylint(), all other files are linted by

20+# test.pep8()

21+PYLINT_FILES=(

22+ searx/preferences.py

23+ searx/testing.py

24+ searx/engines/gigablast.py

25+ searx/engines/deviantart.py

26+ searx/engines/digg.py

27+ searx/engines/google.py

28+ searx/engines/google_news.py

29+ searx/engines/google_videos.py

30+ searx/engines/google_images.py

31+ searx/engines/mediathekviewweb.py

32+ searx/engines/meilisearch.py

33+ searx/engines/solidtorrents.py

34+ searx/engines/solr.py

35+ searx/engines/google_scholar.py

36+ searx/engines/yahoo_news.py

37+ searx/engines/apkmirror.py

38+ searx_extra/update/update_external_bangs.py

39+)

40+

41+PYLINT_SEARX_DISABLE_OPTION="\

42+I,C,R,\

43+W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\

44+E1136"

45+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"

46+PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"

47+

48+help() {

49+ cat <<EOF

50+buildenv

51+ rebuild ./utils/brand.env

52+babel.compile

53+ pybabel compile ./searx/translations

54+data.*

55+ all : update searx/languages.py and ./data/*

56+ languages : update searx/data/engines_languages.json & searx/languages.py

57+ useragents: update searx/data/useragents.json with the most recent versions of Firefox.

58+docs.*

59+ html : build HTML documentation

60+ live : autobuild HTML documentation while editing

61+ gh-pages : deploy on gh-pages branch

62+ prebuild : build reST include files (./${DOCS_BUILD}/includes)

63+ clean : clean documentation build

64+docker

65+ build : build docker image

66+ push : build and push docker image

67+gecko.driver

68+ download & install geckodriver if not already installed (required for

69+ robot_tests)

70+node.*

71+ env : download & install npm dependencies locally

72+ clean : drop npm installations

73+py.*

74+ build : Build python packages at ./${PYDIST}

75+ clean : delete virtualenv and intermediate py files

76+pyenv.* :

77+ install : developer install of searx into virtualenv

78+ uninstall : uninstall developer installation

79+ cmd ... : run command ... in virtualenv

80+ OK : test if virtualenv is OK

81+pypi.upload:

82+ Upload python packages to PyPi (to test use pypi.upload.test)

83+test.* :

84+ pylint : lint PYLINT_FILES, searx/engines, searx & tests

85+ pep8 : pycodestyle (pep8) for all files except PYLINT_FILES

86+ unit : run unit tests

87+ coverage : run unit tests with coverage

88+ robot : run robot test

89+ clean : clean intermediate test stuff

90+themes.* :

91+ all : build all themes

92+ oscar : build oscar theme

93+ simple : build simple theme

94+EOF

95+}

96+

97+

98+if [ "$VERBOSE" = "1" ]; then

99+ SPHINX_VERBOSE="-v"

100+ PYLINT_VERBOSE="-v"

101+fi

102+

103+# needed by sphinx-docs

104+export DOCS_BUILD

105+

106+buildenv() {

107+ SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \

108+ | prefix_stdout "${_Blue}BUILDENV${_creset} "

109+ return "${PIPESTATUS[0]}"

110+}

111+

112+babel.compile() {

113+ build_msg BABEL compile

114+ pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"

115+ dump_return $?

116+}

117+

118+

119+data.all() {

120+ data.languages

121+ data.useragents

122+ build_msg DATA "update searx/data/ahmia_blacklist.txt"

123+ pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py

124+ build_msg DATA "update searx/data/wikidata_units.json"

125+ pyenv.cmd python searx_extra/update/update_wikidata_units.py

126+ build_msg DATA "update searx/data/currencies.json"

127+ pyenv.cmd python searx_extra/update/update_currencies.py

128+}

129+

130+

131+data.languages() {

132+ ( set -e

133+ build_msg ENGINES "fetch languages .."

134+ pyenv.cmd python searx_extra/update/update_languages.py

135+ build_msg ENGINES "update update searx/languages.py"

136+ build_msg DATA "update searx/data/engines_languages.json"

137+ )

138+ dump_return $?

139+}

140+

141+data.useragents() {

142+ build_msg DATA "update searx/data/useragents.json"

143+ pyenv.cmd python searx_extra/update/update_firefox_version.py

144+ dump_return $?

145+}

146+

147+docs.prebuild() {

148+ build_msg DOCS "build ${DOCS_BUILD}/includes"

149+ (

150+ set -e

151+ [ "$VERBOSE" = "1" ] && set -x

152+ mkdir -p "${DOCS_BUILD}/includes"

153+ ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"

154+ ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"

155+ ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"

156+ )

157+ dump_return $?

158+}

159+

160+docker.push() {

161+ docker.build push

162+}

163+

164+# shellcheck disable=SC2119

165+docker.build() {

166+ pyenv.install

167+

168+ build_msg DOCKER build

169+ # run installation in a subprocess and activate pyenv

170+

171+ # See https://www.shellcheck.net/wiki/SC1001 and others ..

172+ # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001

173+ ( set -e

174+ # shellcheck source=/dev/null

175+ source "${PY_ENV_BIN}/activate"

176+

177+ # Check if it is a git repository

178+ if [ ! -d .git ]; then

179+ die 1 "This is not Git repository"

180+ fi

181+ if [ ! -x "$(which git)" ]; then

182+ die 1 "git is not installed"

183+ fi

184+

185+ if ! git remote get-url origin 2> /dev/null; then

186+ die 1 "there is no remote origin"

187+ fi

188+

189+ # This is a git repository

190+

191+ # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)

192+ # awk to remove the "v" and the "g"

193+ SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')

194+

195+ # add the suffix "-dirty" if the repository has uncommited change

196+ # /!\ HACK for searx/searx: ignore utils/brand.env

197+ git update-index -q --refresh

198+ if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then

199+ SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"

200+ fi

201+

202+ # Get the last git commit id, will be added to the Searx version (see Dockerfile)

203+ VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)

204+ build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"

205+

206+ # Check consistency between the git tag and the searx/version.py file

207+ # /! HACK : parse Python file with bash /!

208+ # otherwise it is not possible build the docker image without all Python

209+ # dependencies ( version.py loads __init__.py )

210+ # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")

212+ if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then

213+ err_msg "git tag: $SEARX_GIT_VERSION"

214+ err_msg "searx/version.py: $SEARX_PYTHON_VERSION"

215+ die 1 "Inconsistency between the last git tag and the searx/version.py file"

216+ fi

217+

218+ # define the docker image name

219+ GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/$[^\/]*$.*/\1/')

220+ SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"

221+

222+ # build Docker image

223+ build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

224+ sudo docker build \

225+ --build-arg GIT_URL="${GIT_URL}" \

226+ --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

227+ --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

228+ --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \

229+ --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \

230+ --build-arg LABEL_VCS_URL="${GIT_URL}" \

231+ --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \

232+ --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \

233+ -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

234+

235+ if [ "$1" = "push" ]; then

236+ sudo docker push "${SEARX_IMAGE_NAME}:latest"

237+ sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

238+ fi

239+ )

240+ dump_return $?

241+}

242+

243+# shellcheck disable=SC2119

244+gecko.driver() {

245+ pyenv.install

246+

247+ build_msg INSTALL "gecko.driver"

248+ # run installation in a subprocess and activate pyenv

249+ ( set -e

250+ # shellcheck source=/dev/null

251+ source "${PY_ENV_BIN}/activate"

252+

253+ # TODO : check the current geckodriver version

254+ geckodriver -V > /dev/null 2>&1 || NOTFOUND=1

255+ set +e

256+ if [ -z "$NOTFOUND" ]; then

257+ build_msg INSTALL "geckodriver already installed"

258+ return

259+ fi

260+ PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"

261+ case "$PLATFORM" in

262+ "linux 32bit" | "linux2 32bit") ARCH="linux32";;

263+ "linux 64bit" | "linux2 64bit") ARCH="linux64";;

264+ "windows 32 bit") ARCH="win32";;

265+ "windows 64 bit") ARCH="win64";;

266+ "mac 64bit") ARCH="macos";;

267+ esac

268+ GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";

269+

270+ build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"

271+

272+ FILE="$(mktemp)"

273+ wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver

274+ rm -- "$FILE"

275+ chmod 755 -- "${PY_ENV_BIN}/geckodriver"

276+ )

277+ dump_return $?

278+}

279+

280+node.env() {

281+ local err=0

282+ pushd "${REPO_ROOT}" &> /dev/null

283+ # shellcheck disable=SC2230

284+ which npm &> /dev/null || die 1 'node.env - npm is not found!'

285+

286+ ( set -e

287+ # shellcheck disable=SC2030

288+ PATH="$(npm bin):$PATH"

289+ export PATH

290+

291+ build_msg INSTALL "npm install $NPM_PACKAGES"

292+ # shellcheck disable=SC2086

293+ npm install $NPM_PACKAGES

294+

295+ cd "${REPO_ROOT}/searx/static/themes/oscar"

296+ build_msg INSTALL "($(pwd)) npm install"

297+ npm install

298+

299+ build_msg INSTALL "($(pwd)) npm install"

300+ cd "${REPO_ROOT}/searx/static/themes/simple"

301+ npm install

302+ )

303+ err=$?

304+ popd &> /dev/null

305+ dump_return "$err"

306+}

307+

308+node.clean() {

309+

310+ build_msg CLEAN "locally installed npm dependencies"

311+ rm -rf \

312+ ./node_modules \

313+ ./package-lock.json \

314+ ./searx/static/themes/oscar/package-lock.json \

315+ ./searx/static/themes/oscar/node_modules \

316+ ./searx/static/themes/simple/package-lock.json \

317+ ./searx/static/themes/simple/node_modules

318+ dump_return $?

319+}

320+

321+py.build() {

322+ build_msg BUILD "[pylint] python package ${PYDIST}"

323+ pyenv.cmd python setup.py \

324+ sdist -d "${PYDIST}" \

325+ bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"

326+}

327+

328+py.clean() {

329+ build_msg CLEAN pyenv

330+ ( set -e

331+ pyenv.drop

332+ [ "$VERBOSE" = "1" ] && set -x

333+ rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info

334+ find . -name '*.pyc' -exec rm -f {} +

335+ find . -name '*.pyo' -exec rm -f {} +

336+ find . -name __pycache__ -exec rm -rf {} +

337+ )

338+}

339+

340+pyenv.check() {

341+ cat <<EOF

342+import yaml

343+print('import yaml --> OK')

344+EOF

345+}

346+

347+pyenv.install() {

348+

349+ if ! pyenv.OK; then

350+ py.clean > /dev/null

351+ fi

352+ if pyenv.install.OK > /dev/null; then

353+ return 0

354+ fi

355+ ( set -e

356+ pyenv

357+ build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"

358+ "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"

359+ buildenv

360+ ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"

361+}

362+

363+pyenv.uninstall() {

364+ build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"

365+ pyenv.cmd python setup.py develop --uninstall 2>&1 \

366+ | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "

367+

368+}

369+

370+pypi.upload() {

371+ py.clean

372+ py.build

373+ # https://github.com/pypa/twine

374+ pyenv.cmd twine upload "${PYDIST}"/*

375+}

376+

377+pypi.upload.test() {

378+ py.clean

379+ py.build

380+ pyenv.cmd twine upload -r testpypi "${PYDIST}"/*

381+}

382+

383+test.pylint() {

384+ # shellcheck disable=SC2086

385+ ( set -e

386+ build_msg TEST "[pylint] \$PYLINT_FILES"

387+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

388+ "${PYLINT_FILES[@]}"

389+

390+ build_msg TEST "[pylint] searx/engines"

391+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

392+ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

393+ --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \

394+ searx/engines

395+

396+ build_msg TEST "[pylint] searx tests"

397+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

398+ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

399+ --ignore=searx/engines \

400+ searx tests

401+ )

402+ dump_return $?

403+}

404+

405+test.pep8() {

406+ build_msg TEST 'pycodestyle (formerly pep8)'

407+ local _exclude=""

408+ printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"

409+ pyenv.cmd pycodestyle \

410+ --exclude="searx/static, searx/languages.py, $_exclude " \

411+ --max-line-length=120 \

412+ --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \

413+ searx tests

414+ dump_return $?

415+}

416+

417+test.unit() {

418+ build_msg TEST 'tests/unit'

419+ pyenv.cmd python -m nose2 -s tests/unit

420+ dump_return $?

421+}

422+

423+test.coverage() {

424+ build_msg TEST 'unit test coverage'

425+ ( set -e

426+ pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit

427+ pyenv.cmd coverage report

428+ pyenv.cmd coverage html

429+ )

430+ dump_return $?

431+}

432+

433+test.robot() {

434+ build_msg TEST 'robot'

435+ gecko.driver

436+ PYTHONPATH=. pyenv.cmd python searx/testing.py robot

437+ dump_return $?

438+}

439+

440+test.clean() {

441+ build_msg CLEAN "test stuff"

442+ rm -rf geckodriver.log .coverage coverage/

443+ dump_return $?

444+}

445+

446+themes.all() {

447+ ( set -e

448+ node.env

449+ themes.oscar

450+ themes.simple

451+ )

452+ dump_return $?

453+}

454+

455+themes.oscar() {

456+ local gruntfile=searx/static/themes/oscar/gruntfile.js

457+ build_msg GRUNT "${gruntfile}"

458+ PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"

459+ dump_return $?

460+}

461+

462+themes.simple() {

463+ local gruntfile=searx/static/themes/simple/gruntfile.js

464+ build_msg GRUNT "${gruntfile}"

465+ PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"

466+ dump_return $?

467+}

468+

469+# shellcheck disable=SC2119

470+main() {

471+

472+ local _type

473+ local cmd="$1"; shift

474+

475+ if [ "$cmd" == "" ]; then

476+ help

477+ err_msg "missing command"

478+ return 42

479+ fi

480+

481+ case "$cmd" in

482+ --getenv) var="$1"; echo "${!var}";;

483+ --help) help;;

484+ --*)

485+ help

486+ err_msg "unknown option $cmd"

487+ return 42

488+ ;;

489+ *)

490+ _type="$(type -t "$cmd")"

491+ if [ "$_type" != 'function' ]; then

492+ err_msg "unknown command $1 / use --help"

493+ return 42

494+ else

495+ "$cmd" "$@"

496+ fi

497+ ;;

498+ esac

499+}

500+

501+main "$@"

manage.shdeleted−205

@@ -1,205 +0,0 @@

1−#!/bin/sh

2−

3−export LANG=C

4−

5−BASE_DIR="$(dirname -- "`readlink -f -- "$0"`")"

6−

7−cd -- "$BASE_DIR"

8−set -e

9−

10−# subshell

11−PYTHONPATH="$BASE_DIR"

12−SEARX_DIR="$BASE_DIR/searx"

13−ACTION="$1"

14−

15−. "${BASE_DIR}/utils/brand.env"

16−

17−#

18−# Python

19−#

20−

21−update_packages() {

22− pip install --upgrade pip

23− pip install --upgrade setuptools

24− pip install -Ur "$BASE_DIR/requirements.txt"

25−}

26−

27−update_dev_packages() {

28− update_packages

29− pip install -Ur "$BASE_DIR/requirements-dev.txt"

30−}

31−

32−install_geckodriver() {

33− echo '[!] Checking geckodriver'

34− # TODO : check the current geckodriver version

35− set -e

36− geckodriver -V > /dev/null 2>&1 || NOTFOUND=1

37− set +e

38− if [ -z "$NOTFOUND" ]; then

39− return

40− fi

41− GECKODRIVER_VERSION="v0.28.0"

42− PLATFORM="`python3 -c "import platform; print(platform.system().lower(), platform.architecture()[0])"`"

43− case "$PLATFORM" in

44− "linux 32bit" | "linux2 32bit") ARCH="linux32";;

45− "linux 64bit" | "linux2 64bit") ARCH="linux64";;

46− "windows 32 bit") ARCH="win32";;

47− "windows 64 bit") ARCH="win64";;

48− "mac 64bit") ARCH="macos";;

49− esac

50− GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";

51−

52− if [ -z "$1" ]; then

53− if [ -z "$VIRTUAL_ENV" ]; then

54− printf "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n %s" "$GECKODRIVER_URL"

55− exit

56− else

57− GECKODRIVER_DIR="$VIRTUAL_ENV/bin"

58− fi

59− else

60− GECKODRIVER_DIR="$1"

61− mkdir -p -- "$GECKODRIVER_DIR"

62− fi

63−

64− printf "Installing %s/geckodriver from\n %s" "$GECKODRIVER_DIR" "$GECKODRIVER_URL"

65−

66− FILE="`mktemp`"

67− wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "$GECKODRIVER_DIR" -f "$FILE" geckodriver

68− rm -- "$FILE"

69− chmod 777 -- "$GECKODRIVER_DIR/geckodriver"

70−}

71−

72−locales() {

73− pybabel compile -d "$SEARX_DIR/translations"

74−}

75−

76−

77−#

78−# Web

79−#

80−

81−npm_path_setup() {

82− which npm || (printf 'Error: npm is not found\n'; exit 1)

83− export PATH="$(npm bin)":$PATH

84−}

85−

86−npm_packages() {

87− npm_path_setup

88−

89− echo '[!] install NPM packages'

90− cd -- "$BASE_DIR"

91− npm install less@2.7 less-plugin-clean-css grunt-cli

92−

93− echo '[!] install NPM packages for oscar theme'

94− cd -- "$BASE_DIR/searx/static/themes/oscar"

95− npm install

96−

97− echo '[!] install NPM packages for simple theme'

98− cd -- "$BASE_DIR/searx/static/themes/simple"

99− npm install

100−}

101−

102−docker_build() {

103− # Check if it is a git repository

104− if [ ! -d .git ]; then

105− echo "This is not Git repository"

106− exit 1

107− fi

108−

109− if [ ! -x "$(which git)" ]; then

110− echo "git is not installed"

111− exit 1

112− fi

113−

114− if [ ! git remote get-url origin 2> /dev/null ]; then

115− echo "there is no remote origin"

116− exit 1

117− fi

118−

119− # This is a git repository

120−

121− # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)

122− # awk to remove the "v" and the "g"

123− SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')

124−

125− # add the suffix "-dirty" if the repository has uncommited change

126− # /!\ HACK for searx/searx: ignore utils/brand.env

127− git update-index -q --refresh

128− if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then

129− SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"

130− fi

131−

132− # Get the last git commit id, will be added to the Searx version (see Dockerfile)

133− VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4)

134− echo "Last commit : $VERSION_GITCOMMIT"

135−

136− # Check consistency between the git tag and the searx/version.py file

137− # /!\ HACK : parse Python file with bash /!\

138− # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py )

139− # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")

141− if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then

142− echo "Inconsistency between the last git tag and the searx/version.py file"

143− echo "git tag: $SEARX_GIT_VERSION"

144− echo "searx/version.py: $SEARX_PYTHON_VERSION"

145− exit 1

146− fi

147−

148− # define the docker image name

149− GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/$[^\/]*$.*/\1/')

150− SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"

151−

152− # build Docker image

153− echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

154− sudo docker build \

155− --build-arg GIT_URL="${GIT_URL}" \

156− --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

157− --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

158− --build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \

159− --build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \

160− --build-arg LABEL_VCS_URL="${GIT_URL}" \

161− --build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \

162− --build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \

163− -t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} .

164−

165− if [ "$1" = "push" ]; then

166− sudo docker push ${SEARX_IMAGE_NAME}:latest

167− sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}

168− fi

169−}

170−

171−#

172−# Help

173−#

174−

175−help() {

176− [ -z "$1" ] || printf 'Error: %s\n' "$1"

177− echo "Searx manage.sh help

178−

179−Commands

180−========

181− help - This text

182−

183− Build requirements

184− ------------------

185− update_packages - Check & update production dependency changes

186− update_dev_packages - Check & update development and production dependency changes

187− install_geckodriver - Download & install geckodriver if not already installed (required for robot_tests)

188− npm_packages - Download & install npm dependencies

189−

190− Build

191− -----

192− locales - Compile locales

193−

194−Environment:

195− GIT_URL: ${GIT_URL}

196− ISSUE_URL: ${ISSUE_URL}

197− SEARX_URL: ${SEARX_URL}

198− DOCS_URL: ${DOCS_URL}

199− PUBLIC_INSTANCES: ${PUBLIC_INSTANCES}

200−"

201−}

202−

203−[ "$(command -V "$ACTION" | grep ' function$')" = "" ] \

204− && help "action not found" \

205− || "$ACTION" "$2"

utils/lib.sh+311−4

@@ -86,7 +86,7 @@ set_terminal_colors() {

86 _Red='\e[0;31m'

87 _Green='\e[0;32m'

88 _Yellow='\e[0;33m'

89− _Blue='\e[0;34m'

89+ _Blue='\e[0;94m'

90 _Violet='\e[0;35m'

91 _Cyan='\e[0;36m'

@@ -95,12 +95,12 @@ set_terminal_colors() {

95 _BRed='\e[1;31m'

96 _BGreen='\e[1;32m'

97 _BYellow='\e[1;33m'

98− _BBlue='\e[1;34m'

98+ _BBlue='\e[1;94m'

99 _BPurple='\e[1;35m'

100 _BCyan='\e[1;36m'

101}

102

103−if [ ! -p /dev/stdout ]; then

103+if [ ! -p /dev/stdout ] && [ ! "$TERM" = 'dumb' ] && [ ! "$TERM" = 'unknown' ]; then

104 set_terminal_colors

105fi

106

@@ -152,6 +152,22 @@ err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; }

152warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; }

153info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; }

154

155+build_msg() {

156+ local tag="$1 "

157+ shift

158+ echo -e "${_Blue}${tag:0:10}${_creset}$*"

159+}

160+

161+dump_return() {

162+

163+ # Use this as last command in your function to prompt an ERROR message if

164+ # the exit code is not zero.

165+

166+ local err=$1

167+ [ "$err" -ne "0" ] && err_msg "${FUNCNAME[1]} exit with error ($err)"

168+ return "$err"

169+}

170+

155clean_stdin() {

171clean_stdin() {

156 if [[ $(uname -s) != 'Darwin' ]]; then

172 if [[ $(uname -s) != 'Darwin' ]]; then

157 while read -r -n1 -t 0.1; do : ; done

173 while read -r -n1 -t 0.1; do : ; done

@@ -496,6 +512,291 @@ service_is_available() {

496 return "$exit_val"

512 return "$exit_val"

497}

513}

498

514

515+# python

516+# ------

517+

518+PY="${PY:=3}"

519+PYTHON="${PYTHON:=python$PY}"

520+PY_ENV="${PY_ENV:=local/py${PY}}"

521+PY_ENV_BIN="${PY_ENV}/bin"

522+PY_ENV_REQ="${PY_ENV_REQ:=${REPO_ROOT}/requirements*.txt}"

523+

524+# List of python packages (folders) or modules (files) installed by command:

525+# pyenv.install

526+PYOBJECTS="${PYOBJECTS:=.}"

527+

528+# folder where the python distribution takes place

529+PYDIST="${PYDIST:=dist}"

530+

531+# folder where the intermediate build files take place

532+PYBUILD="${PYBUILD:=build/py${PY}}"

533+

534+# https://www.python.org/dev/peps/pep-0508/#extras

535+#PY_SETUP_EXTRAS='[develop,test]'

536+PY_SETUP_EXTRAS="${PY_SETUP_EXTRAS:=[develop,test]}"

537+

538+PIP_BOILERPLATE=( pip wheel setuptools )

539+

540+# shellcheck disable=SC2120

541+pyenv() {

542+

543+ # usage: pyenv [vtenv_opts ...]

544+ #

545+ # vtenv_opts: see 'pip install --help'

546+ #

547+ # Builds virtualenv with 'requirements*.txt' (PY_ENV_REQ) installed. The

548+ # virtualenv will be reused by validating sha256sum of the requirement

549+ # files.

550+

551+ required_commands \

552+ sha256sum "${PYTHON}" \

553+ || exit

554+

555+ local pip_req=()

556+

557+ if ! pyenv.OK > /dev/null; then

558+ rm -f "${PY_ENV}/${PY_ENV_REQ}.sha256"

559+ pyenv.drop > /dev/null

560+ build_msg PYENV "[virtualenv] installing ${PY_ENV_REQ} into ${PY_ENV}"

561+

562+ "${PYTHON}" -m venv "$@" "${PY_ENV}"

563+ "${PY_ENV_BIN}/python" -m pip install -U "${PIP_BOILERPLATE[@]}"

564+

565+ for i in ${PY_ENV_REQ}; do

566+ pip_req=( "${pip_req[@]}" "-r" "$i" )

567+ done

568+

569+ (

570+ [ "$VERBOSE" = "1" ] && set -x

571+ # shellcheck disable=SC2086

572+ "${PY_ENV_BIN}/python" -m pip install "${pip_req[@]}" \

573+ && sha256sum ${PY_ENV_REQ} > "${PY_ENV}/requirements.sha256"

574+ )

575+ fi

576+ pyenv.OK

577+}

578+

579+_pyenv_OK=''

580+pyenv.OK() {

581+

582+ # probes if pyenv exists and runs the script from pyenv.check

583+

584+ [ "$_pyenv_OK" == "OK" ] && return 0

585+

586+ if [ ! -f "${PY_ENV_BIN}/python" ]; then

587+ build_msg PYENV "[virtualenv] missing ${PY_ENV_BIN}/python"

588+ return 1

589+ fi

590+

591+ if [ ! -f "${PY_ENV}/requirements.sha256" ] \

592+ || ! sha256sum --check --status <"${PY_ENV}/requirements.sha256" 2>/dev/null; then

593+ build_msg PYENV "[virtualenv] requirements.sha256 failed"

594+ sed 's/^/ [virtualenv] - /' <"${PY_ENV}/requirements.sha256"

595+ return 1

596+ fi

597+

598+ pyenv.check \

599+ | "${PY_ENV_BIN}/python" 2>&1 \

600+ | prefix_stdout "${_Blue}PYENV ${_creset}[check] "

601+

602+ local err=${PIPESTATUS[1]}

603+ if [ "$err" -ne "0" ]; then

604+ build_msg PYENV "[check] python test failed"

605+ return "$err"

606+ fi

607+

608+ build_msg PYENV "OK"

609+ _pyenv_OK="OK"

610+ return 0

611+}

612+

613+pyenv.drop() {

614+

615+ build_msg PYENV "[virtualenv] drop ${PY_ENV}"

616+ rm -rf "${PY_ENV}"

617+ _pyenv_OK=''

618+

619+}

620+

621+pyenv.check() {

622+

623+ # Prompts a python script with additional checks. Used by pyenv.OK to check

624+ # if virtualenv is ready to install python objects. This function should be

625+ # overwritten by the application script.

626+

627+ local imp=""

628+

629+ for i in "${PIP_BOILERPLATE[@]}"; do

630+ imp="$imp, $i"

631+ done

632+

633+ cat <<EOF

634+import ${imp#,*}

635+

636+EOF

637+}

638+

639+pyenv.install() {

640+

641+ if ! pyenv.OK; then

642+ py.clean > /dev/null

643+ fi

644+ if ! pyenv.install.OK > /dev/null; then

645+ build_msg PYENV "[install] ${PYOBJECTS}"

646+ if ! pyenv.OK >/dev/null; then

647+ pyenv

648+ fi

649+ for i in ${PYOBJECTS}; do

650+ build_msg PYENV "[install] pip install -e '$i${PY_SETUP_EXTRAS}'"

651+ "${PY_ENV_BIN}/python" -m pip install -e "$i${PY_SETUP_EXTRAS}"

652+ done

653+ fi

654+ pyenv.install.OK

655+}

656+

657+_pyenv_install_OK=''

658+pyenv.install.OK() {

659+

660+ [ "$_pyenv_install_OK" == "OK" ] && return 0

661+

662+ local imp=""

663+ local err=""

664+

665+ if [ "." = "${PYOBJECTS}" ]; then

666+ imp="import $(basename "$(pwd)")"

667+ else

668+ # shellcheck disable=SC2086

669+ for i in ${PYOBJECTS}; do imp="$imp, $i"; done

670+ imp="import ${imp#,*} "

671+ fi

672+ (

673+ [ "$VERBOSE" = "1" ] && set -x

674+ "${PY_ENV_BIN}/python" -c "import sys; sys.path.pop(0); $imp;" 2>/dev/null

675+ )

676+

677+ err=$?

678+ if [ "$err" -ne "0" ]; then

679+ build_msg PYENV "[install] python installation test failed"

680+ return "$err"

681+ fi

682+

683+ build_msg PYENV "[install] OK"

684+ _pyenv_install_OK="OK"

685+ return 0

686+}

687+

688+pyenv.uninstall() {

689+

690+ build_msg PYENV "[uninstall] ${PYOBJECTS}"

691+

692+ if [ "." = "${PYOBJECTS}" ]; then

693+ pyenv.cmd python setup.py develop --uninstall 2>&1 \

694+ | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "

695+ else

696+ pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 \

697+ | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "

698+ fi

699+}

700+

701+

702+pyenv.cmd() {

703+ pyenv.install

704+ ( set -e

705+ # shellcheck source=/dev/null

706+ source "${PY_ENV_BIN}/activate"

707+ [ "$VERBOSE" = "1" ] && set -x

708+ "$@"

709+ )

710+}

711+

712+# Sphinx doc

713+# ----------

714+

715+GH_PAGES="build/gh-pages"

716+DOCS_DIST="${DOCS_DIST:=dist/docs}"

717+DOCS_BUILD="${DOCS_BUILD:=build/docs}"

718+

719+docs.html() {

720+ build_msg SPHINX "HTML ./docs --> file://$(readlink -e "$(pwd)/$DOCS_DIST")"

721+ pyenv.install

722+ docs.prebuild

723+ # shellcheck disable=SC2086

724+ PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-build \

725+ ${SPHINX_VERBOSE} ${SPHINXOPTS} \

726+ -b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}"

727+ dump_return $?

728+}

729+

730+docs.live() {

731+ build_msg SPHINX "autobuild ./docs --> file://$(readlink -e "$(pwd)/$DOCS_DIST")"

732+ pyenv.install

733+ docs.prebuild

734+ # shellcheck disable=SC2086

735+ PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-autobuild \

736+ ${SPHINX_VERBOSE} ${SPHINXOPTS} --open-browser --host 0.0.0.0 \

737+ -b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}"

738+ dump_return $?

739+}

740+

741+docs.clean() {

742+ build_msg CLEAN "docs -- ${DOCS_BUILD} ${DOCS_DIST}"

743+ # shellcheck disable=SC2115

744+ rm -rf "${GH_PAGES}" "${DOCS_BUILD}" "${DOCS_DIST}"

745+ dump_return $?

746+}

747+

748+docs.prebuild() {

749+ # Dummy function to run some actions before sphinx-doc build gets started.

750+ # This finction needs to be overwritten by the application script.

751+ true

752+ dump_return $?

753+}

754+

755+# shellcheck disable=SC2155

756+docs.gh-pages() {

757+

758+ # The commit history in the gh-pages branch makes no sense, the history only

759+ # inflates the repository unnecessarily. Therefore a *new orphan* branch

760+ # is created each time we deploy on the gh-pages branch.

761+

762+ docs.clean

763+ docs.prebuild

764+ docs.html

765+

766+ [ "$VERBOSE" = "1" ] && set -x

767+ local head="$(git rev-parse HEAD)"

768+ local branch="$(git name-rev --name-only HEAD)"

769+ local remote="$(git config branch."${branch}".remote)"

770+ local remote_url="$(git config remote."${remote}".url)"

771+

772+ # prepare the *orphan* gh-pages working tree

773+ (

774+ git worktree remove -f "${GH_PAGES}"

775+ git branch -D gh-pages

776+ ) &> /dev/null || true

777+ git worktree add --no-checkout "${GH_PAGES}" origin/master

778+

779+ pushd "${GH_PAGES}" &> /dev/null

780+ git checkout --orphan gh-pages

781+ git rm -rfq .

782+ popd &> /dev/null

783+

784+ cp -r "${DOCS_DIST}"/* "${GH_PAGES}"/

785+ touch "${GH_PAGES}/.nojekyll"

786+ cat > "${GH_PAGES}/404.html" <<EOF

787+<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>

788+EOF

789+

790+ pushd "${GH_PAGES}" &> /dev/null

791+ git add --all .

792+ git commit -q -m "gh-pages build from: ${branch}@${head} (${remote_url})"

793+ git push -f "${remote}" gh-pages

794+ popd &> /dev/null

795+

796+ set +x

797+ build_msg GH-PAGES "deployed"

798+}

799+

499# golang

800# golang

500# ------

801# ------

501

802

@@ -1250,7 +1551,7 @@ pkg_install() {

1250 centos)

1551 centos)

1251 # shellcheck disable=SC2068

1552 # shellcheck disable=SC2068

1252 yum install -y $@

1553 yum install -y $@

1253− ;;

1554+ ;;

1254 esac

1555 esac

1255}

1556}

1256

1557

@@ -1382,6 +1683,12 @@ LXC_ENV_FOLDER=

1382if in_container; then

1683if in_container; then

1383 # shellcheck disable=SC2034

1684 # shellcheck disable=SC2034

1384 LXC_ENV_FOLDER="lxc-env/$(hostname)/"

1685 LXC_ENV_FOLDER="lxc-env/$(hostname)/"

1686+ PY_ENV="${LXC_ENV_FOLDER}${PY_ENV}"

1687+ PY_ENV_BIN="${LXC_ENV_FOLDER}${PY_ENV_BIN}"

1688+ PYDIST="${LXC_ENV_FOLDER}${PYDIST}"

1689+ PYBUILD="${LXC_ENV_FOLDER}${PYBUILD}"

1690+ DOCS_DIST="${LXC_ENV_FOLDER}${DOCS_DIST}"

1691+ DOCS_BUILD="${LXC_ENV_FOLDER}${DOCS_BUILD}"

1385fi

1692fi

1386

1693

1387lxc_init_container_env() {

1694lxc_init_container_env() {

utils/lxc.sh+2−2

@@ -142,11 +142,11 @@ main() {

142 local _usage="unknown or missing $1 command $2"

143

144 # don't check prerequisite when in recursion

145− if [[ ! $1 == __* ]]; then

145+ if [[ ! $1 == __* ]] && [[ ! $1 == --help ]]; then

146 if ! in_container; then

147 ! required_commands lxc && lxd_info && exit 42

148 fi

149− [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42

149+ [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42

150 fi

151

152 case $1 in

utils/makefile.include+10−49

@@ -1,4 +1,5 @@

1# -*- coding: utf-8; mode: makefile-gmake -*-

2+# SPDX-License-Identifier: AGPL-3.0-or-later

3ifeq (,$(wildcard /.lxcenv.mk))

4ifeq (,$(wildcard /.lxcenv.mk))

4PHONY += lxc-activate lxc-purge

5PHONY += lxc-activate lxc-purge

@@ -10,60 +11,26 @@ else

10 include /.lxcenv.mk

11 include /.lxcenv.mk

11endif

12endif

14+PHONY += make-help

13ifeq (,$(wildcard /.lxcenv.mk))

15ifeq (,$(wildcard /.lxcenv.mk))

14make-help:

16make-help:

15else

17else

16make-help: lxc-help

18make-help: lxc-help

17endif

19endif

20+ @echo 'options:'

18 @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'

21 @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build'

19 @echo ' make V=2 [targets] 2 => give reason for rebuild of target'

22 @echo ' make V=2 [targets] 2 => give reason for rebuild of target'

21−quiet_cmd_common_clean = CLEAN $@

22− cmd_common_clean = \

23− find . -name '*.orig' -exec rm -f {} + ;\

24− find . -name '*.rej' -exec rm -f {} + ;\

25− find . -name '*~' -exec rm -f {} + ;\

26− find . -name '*.bak' -exec rm -f {} + ;\

27−

28−FMT = cat

29−ifeq ($(shell which fmt >/dev/null 2>&1; echo $$?), 0)

30−FMT = fmt

31−endif

32−

33−# MS-Windows

34−#

35−# For a minimal *make-environment*, I'am using the gnu-tools from:

36−#

37−# - GNU MCU Eclipse Windows Build Tools, which brings 'make', 'rm' etc.

38−# https://github.com/gnu-mcu-eclipse/windows-build-tools/releases

39−#

40−# - git for Windows, which brings 'find', 'grep' etc.

41−# https://git-scm.com/download/win

42−

43−

44−# normpath

45−#

46−# System-dependent normalization of the path name

47−#

48−# usage: $(call normpath,/path/to/file)

49−

50−normpath = $1

51−ifeq ($(OS),Windows_NT)

52− normpath = $(subst /,\,$1)

53−endif

54−

55−

56−# stolen from linux/Makefile

57−#

58−

59ifeq ("$(origin V)", "command line")

24ifeq ("$(origin V)", "command line")

60− KBUILD_VERBOSE = $(V)

25+ VERBOSE = $(V)

61endif

26endif

62−ifndef KBUILD_VERBOSE

27+ifndef VERBOSE

63− KBUILD_VERBOSE = 0

28+ VERBOSE = 0

64endif

29endif

66−ifeq ($(KBUILD_VERBOSE),1)

31+export VERBOSE

32+

33+ifeq ($(VERBOSE),1)

67 quiet =

34 quiet =

68 Q =

35 Q =

69else

36else

@@ -75,14 +42,8 @@ endif

75#

42#

77# Convenient variables

44# Convenient variables

78−comma := ,

79−quote := "

80−#" this comment is only for emacs highlighting

81squote := '

45squote := '

82#' this comment is only for emacs highlighting

46#' this comment is only for emacs highlighting

83−empty :=

84−space := $(empty) $(empty)

85−space_escape := _-_SPACE_-_

87# Find any prerequisites that is newer than target or that does not exist.

48# Find any prerequisites that is newer than target or that does not exist.

88# PHONY targets skipped in both cases.

49# PHONY targets skipped in both cases.

@@ -107,7 +68,7 @@ any-prereq = $(filter-out $(PHONY),$?) $(filter-out $(PHONY) $(wildcard $^),$^)

107# (5) No dir/.target.cmd file (used to store command line)

68# (5) No dir/.target.cmd file (used to store command line)

108# (6) No dir/.target.cmd file and target not listed in $(targets)

69# (6) No dir/.target.cmd file and target not listed in $(targets)

109# This is a good hint that there is a bug in the kbuild file

70# This is a good hint that there is a bug in the kbuild file

110−ifeq ($(KBUILD_VERBOSE),2)

71+ifeq ($(VERBOSE),2)

111why = \

72why = \

112 $(if $(filter $@, $(PHONY)),- due to target is PHONY, \

73 $(if $(filter $@, $(PHONY)),- due to target is PHONY, \

113 $(if $(wildcard $@), \

74 $(if $(wildcard $@), \

utils/makefile.pythondeleted−269

@@ -1,269 +0,0 @@

1−# -*- coding: utf-8; mode: makefile-gmake -*-

2−

3−# list of python packages (folders) or modules (files) of this build

4−PYOBJECTS ?=

5−

6−SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python

7−export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH

8−export PY_ENV PYDIST PYBUILD

9−

10−# folder where the python distribution takes place

11−PYDIST = ./$(LXC_ENV_FOLDER)dist

12−# folder where the python intermediate build files take place

13−PYBUILD = ./$(LXC_ENV_FOLDER)build

14−# python version to use

15−PY ?=3

16−# $(PYTHON) points to the python interpreter from the OS! The python from the

17−# OS is needed e.g. to create a virtualenv. For tasks inside the virtualenv the

18−# interpeter from '$(PY_ENV_BIN)/python' is used.

19−PYTHON ?= python$(PY)

20−PIP ?= pip$(PY)

21−PIP_INST ?= --user

22−

23−# https://www.python.org/dev/peps/pep-0508/#extras

24−#PY_SETUP_EXTRAS ?= \[develop,test\]

25−PY_SETUP_EXTRAS ?=

26−

27−PYDEBUG ?= --pdb

28−PYLINT_RC ?= .pylintrc

29−

30−TEST_FOLDER ?= ./tests

31−TEST ?= .

32−

33−PY_ENV = ./$(LXC_ENV_FOLDER)local/py$(PY)

34−PY_ENV_BIN = $(PY_ENV)/bin

35−PY_ENV_ACT = . $(PY_ENV_BIN)/activate

36−

37−ifeq ($(OS),Windows_NT)

38− PYTHON = python

39− PY_ENV_BIN = $(PY_ENV)/Scripts

40− PY_ENV_ACT = $(PY_ENV_BIN)/activate

41−endif

42−

43−VTENV_OPTS ?=

44−

45−python-help::

46− @echo 'makefile.python:'

47− @echo ' pyenv | pyenv[un]install'

48− @echo ' build $(PY_ENV) & [un]install python objects'

49− @echo ' targts using pyenv $(PY_ENV):'

50− @echo ' pylint - run pylint *linting*'

51− @echo ' pytest - run *tox* test on python objects'

52− @echo ' pydebug - run tests within a PDB debug session'

53− @echo ' pybuild - build python packages ($(PYDIST) $(PYBUILD))'

54− @echo ' pyclean - clean intermediate python objects'

55− @echo ' targets using system users environment:'

56− @echo ' py[un]install - [un]install python objects in editable mode'

57− @echo ' upload-pypi - upload $(PYDIST)/* files to PyPi'

58− @echo 'options:'

59− @echo ' make PY=3.7 [targets] => to eval targets with python 3.7 ($(PY))'

60− @echo ' make PIP_INST= => to set/unset pip install options ($(PIP_INST))'

61− @echo ' make TEST=. => choose test from $(TEST_FOLDER) (default "." runs all)'

62− @echo ' make DEBUG= => target "debug": do not invoke PDB on errors'

63− @echo ' make PY_SETUP_EXTRAS => also install extras_require from setup.py \[develop,test\]'

64− @echo ' when using target "pydebug", set breakpoints within py-source by adding::'

65− @echo ' DEBUG()'

66−

67−# ------------------------------------------------------------------------------

68−# OS requirements

69−# ------------------------------------------------------------------------------

70−

71−PHONY += msg-python-exe python-exe

72−msg-python-exe:

73− @echo "\n $(PYTHON) is required\n\n\

74− Make sure you have $(PYTHON) installed, grab it from\n\

75− https://www.python.org or install it from your package\n\

76− manager. On debian based OS these requirements are\n\

77− installed by::\n\n\

78− sudo -H add-apt-repository ppa:deadsnakes/ppa\n\

79− sudo -H apt update\n\

80− sudo -H apt-get install $(PYTHON) $(PYTHON)-venv\n"

81−

82−ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1)

83−python-exe: msg-python-exe

84− $(error The '$(PYTHON)' command was not found)

85−else

86−python-exe:

87− @:

88−endif

89−

90−msg-pip-exe:

91− @echo "\n $(PIP) is required\n\n\

92− Make sure you have updated pip installed, grab it from\n\

93− https://pip.pypa.io or install it from your package\n\

94− manager. On debian based OS these requirements are\n\

95− installed by::\n\n\

96− sudo -H apt-get install python$(PY)-pip\n" | $(FMT)

97−

98−ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1)

99−pip-exe: msg-pip-exe

100− $(error The '$(PIP)' command was not found)

101−else

102−pip-exe:

103− @:

104−endif

105−

106−# ------------------------------------------------------------------------------

107−# commands

108−# ------------------------------------------------------------------------------

109−

110−# $2 path to folder with setup.py, this uses pip from the OS

111−quiet_cmd_pyinstall = INSTALL $2

112− cmd_pyinstall = $(PIP) $(PIP_VERBOSE) install $(PIP_INST) -e $2$(PY_SETUP_EXTRAS)

113−

114−# $2 path to folder with setup.py, this uses pip from pyenv (not OS!)

115−quiet_cmd_pyenvinstall = PYENV install $2

116− cmd_pyenvinstall = \

117− if ! cat $(PY_ENV)/requirements.sha256 2>/dev/null | sha256sum --check --status 2>/dev/null; then \

118− rm -f $(PY_ENV)/requirements.sha256; \

119− $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) &&\

120− sha256sum requirements*.txt > $(PY_ENV)/requirements.sha256 ;\

121− else \

122− echo "PYENV $2 already installed"; \

123− fi

124−

125−# Uninstall the package. Since pip does not uninstall the no longer needed

126−# depencies (something like autoremove) the depencies remain.

127−

128−# $2 package name to uninstall, this uses pip from the OS.

129−quiet_cmd_pyuninstall = UNINSTALL $2

130− cmd_pyuninstall = $(PIP) $(PIP_VERBOSE) uninstall --yes $2

131−

132−# $2 path to folder with setup.py, this uses pip from pyenv (not OS!)

133−quiet_cmd_pyenvuninstall = PYENV uninstall $2

134− cmd_pyenvuninstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes $2

135−

136−# $2 path to folder where virtualenv take place

137−quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate

138− cmd_virtualenv = \

139− if [ -d "./$(PY_ENV)" -a -x "./$(PY_ENV_BIN)/python" ]; then \

140− echo "PYENV using virtualenv from $2"; \

141− else \

142− $(PYTHON) -m venv $(VTENV_OPTS) $2; \

143− $(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel setuptools; \

144− $(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt; \

145− fi

146−

147−# $2 path to lint

148−quiet_cmd_pylint = LINT $@

149− cmd_pylint = $(PY_ENV_BIN)/python -m pylint -j 0 --rcfile $(PYLINT_RC) $2

150−

151−quiet_cmd_pytest = TEST $@

152− cmd_pytest = $(PY_ENV_BIN)/python -m tox -vv

153−

154−# setuptools, pip, easy_install its a mess full of cracks, a documentation hell

155−# and broken by design ... all sucks, I really, really hate all this ... aaargh!

156−#

157−# About python packaging see `Python Packaging Authority`_. Most of the names

158−# here are mapped to ``setup(<name1>=..., <name2>=...)`` arguments in

159−# ``setup.py``. See `Packaging and distributing projects`_ about ``setup(...)``

160−# arguments. If this is all new for you, start with `PyPI Quick and Dirty`_.

161−#

162−# Further read:

163−#

164−# - pythonwheels_

165−# - setuptools_

166−# - packaging_

167−# - sdist_

168−# - installing_

169−#

170−# .. _`Python Packaging Authority`: https://www.pypa.io

171−# .. _`Packaging and distributing projects`: https://packaging.python.org/guides/distributing-packages-using-setuptools/

172−# .. _`PyPI Quick and Dirty`: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/

173−# .. _pythonwheels: https://pythonwheels.com/

174−# .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html

175−# .. _packaging: https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-and-distributing-projects

176−# .. _sdist: https://packaging.python.org/guides/distributing-packages-using-setuptools/#source-distributions

177−# .. _bdist_wheel: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pure-python-wheels

178−# .. _installing: https://packaging.python.org/tutorials/installing-packages/

179−#

180−quiet_cmd_pybuild = BUILD $@

181− cmd_pybuild = $(PY_ENV_BIN)/python setup.py \

182− sdist -d $(PYDIST) \

183− bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST)

184−

185−quiet_cmd_pyclean = CLEAN $@

186−# remove 'build' folder since bdist_wheel does not care the --bdist-dir

187− cmd_pyclean = \

188− rm -rf $(PYDIST) $(PYBUILD) $(PY_ENV) ./.tox *.egg-info ;\

189− find . -name '*.pyc' -exec rm -f {} + ;\

190− find . -name '*.pyo' -exec rm -f {} + ;\

191− find . -name __pycache__ -exec rm -rf {} +

192−

193−# ------------------------------------------------------------------------------

194−# targets

195−# ------------------------------------------------------------------------------

196−

197−# for installation use the pip from the OS!

198−PHONY += pyinstall

199−pyinstall: pip-exe

200− $(call cmd,pyinstall,.)

201−

202−PHONY += pyuninstall

203−pyuninstall: pip-exe

204− $(call cmd,pyuninstall,$(PYOBJECTS))

205−

206−# for installation use the pip from PY_ENV (not the OS)!

207−PHONY += pyenvinstall

208−pyenvinstall: $(PY_ENV)

209− $(call cmd,pyenvinstall,.)

210−

211−PHONY += pyenvuninstall

212−pyenvuninstall: $(PY_ENV)

213− $(call cmd,pyenvuninstall,$(PYOBJECTS))

214−

215−PHONY += pyclean

216−pyclean:

217− $(call cmd,pyclean)

218−

219−# to build *local* environment, python from the OS is needed!

220−pyenv: $(PY_ENV)

221−$(PY_ENV): python-exe

222− $(call cmd,virtualenv,$(PY_ENV))

223−

224−PHONY += pylint-exe

225−pylint-exe: $(PY_ENV)

226− @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install pylint

227−

228−PHONY += pylint

229−pylint: pylint-exe

230− $(call cmd,pylint,$(PYOBJECTS))

231−

232−PHONY += pybuild

233−pybuild: $(PY_ENV)

234− $(call cmd,pybuild)

235−

236−PHONY += pytest

237−pytest: $(PY_ENV)

238− $(call cmd,pytest)

239−

240−PHONY += pydebug

241−# set breakpoint with:

242−# DEBUG()

243−# e.g. to run tests in debug mode in emacs use:

244−# 'M-x pdb' ... 'make pydebug'

245−pydebug: $(PY_ENV)

246− DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST)

247−

248−# runs python interpreter from ./local/py<N>/bin/python

249−pyenv-python: pyenvinstall

250− $(PY_ENV_BIN)/python -i

251−

252−# With 'dependency_links=' setuptools supports dependencies on packages hosted

253−# on other reposetories then PyPi, see "Packages Not On PyPI" [1]. The big

254−# drawback is, due to security reasons (I don't know where the security gate on

255−# PyPi is), this feature is not supported by pip [2]. Thats why an upload to

256−# PyPi is required and since uploads via setuptools is not recommended, we have

257−# to imstall / use twine ... its really a mess.

258−#

259−# [1] https://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi

260−# [2] https://github.com/pypa/pip/pull/1519

261−

262−# https://github.com/pypa/twine

263−PHONY += upload-pypi upload-pypi-test

264−upload-pypi: pyclean pyenvinstall pybuild

265− @$(PY_ENV_BIN)/twine upload $(PYDIST)/*

266−

267−upload-pypi-test: pyclean pyenvinstall pybuild

268− @$(PY_ENV_BIN)/twine upload -r testpypi $(PYDIST)/*

269−.PHONY: $(PHONY)

utils/makefile.sphinxdeleted−199

@@ -1,199 +0,0 @@

1−# -*- coding: utf-8; mode: makefile-gmake -*-

2−

3−export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST

4−

5−# You can set these variables from the command line.

6−SPHINXOPTS ?=

7−SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build

8−SPHINX_CONF ?= conf.py

9−

10−DOCS_FOLDER = ./docs

11−DOCS_BUILD = ./$(LXC_ENV_FOLDER)build/docs

12−DOCS_DIST = ./$(LXC_ENV_FOLDER)dist/docs

13−GH_PAGES ?= build/gh-pages

14−

15−BOOKS_FOLDER = ./docs

16−BOOKS_DIST = ./$(LXC_ENV_FOLDER)dist/books

17−

18−ifeq ($(KBUILD_VERBOSE),1)

19− SPHINX_VERBOSE = "-v"

20−else

21− SPHINX_VERBOSE =

22−endif

23−

24−

25−docs-help:

26− @echo 'makefile.sphinx:'

27− @echo ' docs-clean - clean intermediate doc objects'

28− @echo ' $(GH_PAGES) - create & upload github pages'

29− @echo ' sphinx-pdf - run sphinx latex & pdf targets'

30− @echo ''

31− @echo ' books/{name}.html : build only the HTML of document {name}'

32− @echo ' valid values for books/{name}.html are:'

33− @echo ' $(BOOKS_HTML)' | $(FMT)

34− @echo ' books/{name}.pdf : build only the PDF of document {name}'

35− @echo ' valid values for books/{name}.pdf are:'

36− @echo ' $(BOOKS_PDF) ' | $(FMT)

37−

38−# ------------------------------------------------------------------------------

39−# requirements

40−# ------------------------------------------------------------------------------

41−

42−PHONY += msg-texlive texlive

43−

44−ifeq ($(shell which xelatex >/dev/null 2>&1; echo $$?), 1)

45−texlive: msg-TeXLive

46− $(error The 'xelatex' command was not found)

47−else

48−texlive:

49− @:

50−endif

51−

52−msg-texlive:

53− $(Q)echo "\n\

54−The TeX/PDF output and the *math* extension require TexLive and latexmk:\n\n\

55− Make sure you have a updated TeXLive with XeTeX engine installed, grab it\n\

56− it from https://www.tug.org/texlive or install it from your package manager.\n\n\

57− Install latexmk from your package manager or visit https://ctan.org/pkg/latexmk\n\n\

58− Sphinx-doc produce (Xe)LaTeX files which might use additional TeX-packages\n\

59− and fonts. To process these LaTeX files, a TexLive installation with the\n\

60− additional packages is required. On debian based OS these requirements\n\

61− are installed by::\n\n\

62− sudo -H apt-get install\n\

63− latexmk\n\

64− texlive-base texlive-xetex texlive-latex-recommended\n\

65− texlive-extra-utils dvipng ttf-dejavu\n"

66−

67−# ------------------------------------------------------------------------------

68−# commands

69−# ------------------------------------------------------------------------------

70−

71−# $2 sphinx builder e.g. "html"

72−# $3 path where configuration file (conf.py) is located

73−# $4 sourcedir

74−# $5 dest subfolder e.g. "man" for man pages at $(DOCS_DIST)/man

75−

76−quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(DOCS_DIST)/$5)

77− cmd_sphinx = SPHINX_CONF=$(abspath $4/$(SPHINX_CONF))\

78− $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\

79− -b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5

80−

81−quiet_cmd_sphinx_autobuild = SPHINX $@ --> file://$(abspath $(DOCS_DIST)/$5)

82− cmd_sphinx_autobuild = PATH="$(PY_ENV_BIN):$(PATH)" $(PY_ENV_BIN)/sphinx-autobuild $(SPHINX_VERBOSE) --open-browser --host 0.0.0.0 $(SPHINXOPTS)\

83− -b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5

84−

85−quiet_cmd_sphinx_clean = CLEAN $@

86− cmd_sphinx_clean = rm -rf $(DOCS_BUILD) $(DOCS_DIST) $(GH_PAGES)/* $(GH_PAGES)/.buildinfo

87−

88−# ------------------------------------------------------------------------------

89−# targets

90−# ------------------------------------------------------------------------------

91−

92−# build PDF of whole documentation in: $(DOCS_DIST)/pdf

93−

94−PHONY += sphinx-pdf

95−sphinx-pdf: sphinx-latex

96− $(Q)cd $(DOCS_BUILD)/latex/; make all-pdf

97− $(Q)mkdir -p $(DOCS_DIST)/pdf

98− $(Q)cp $(DOCS_BUILD)/latex/*.pdf $(DOCS_DIST)/pdf

99− @echo "SPHINX *.pdf --> file://$(abspath $(DOCS_DIST)/pdf)"

100−

101−PHONY += sphinx-latex

102−sphinx-latex: pyenvinstall texlive

103− $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\

104− -b latex \

105− -c $(DOCS_FOLDER) \

106− -d $(DOCS_BUILD)/.doctrees \

107− $(DOCS_FOLDER) \

108− $(DOCS_BUILD)/latex

109−

110−# Sphinx projects, we call them *books* (what is more common). Books are

111−# folders under $(BOOKS_FOLDER) containing a conf.py file. The HTML output goes

112−# to folder $(BOOKS_DIST)/<name> while PDF is placed (BOOKS_DIST)/<name>/pdf

113−

114−BOOKS=$(patsubst $(BOOKS_FOLDER)/%/conf.py,books/%,$(wildcard $(BOOKS_FOLDER)/*/conf.py))

115−

116−# fine grained targets

117−BOOKS_HTML = $(patsubst %,%.html,$(BOOKS))

118−BOOKS_CLEAN = $(patsubst %,%.clean,$(BOOKS))

119−BOOKS_LATEX = $(patsubst %,%.latex,$(BOOKS))

120−BOOKS_PDF = $(patsubst %,%.pdf,$(BOOKS))

121−BOOKS_LIVE = $(patsubst %,%.live,$(BOOKS))

122−

123−$(BOOKS_DIST):

124− mkdir -p $(BOOKS_DIST)

125−

126−PHONY += $(BOOKS_HTML)

127−$(BOOKS_HTML): pyenvinstall | $(BOOKS_DIST)

128− SPHINX_CONF=$(patsubst books/%.html,%,$@)/conf.py \

129− $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\

130− -b html \

131− -c $(DOCS_FOLDER) \

132− -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \

133− $(BOOKS_FOLDER)/$(patsubst books/%.html,%,$@) \

134− $(BOOKS_DIST)/$(patsubst books/%.html,%,$@)

135− @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))"

136−

137−PHONY += $(BOOKS_HTML)

138−$(BOOKS_LIVE): pyenvinstall | $(BOOKS_DIST)

139− PATH="$(PY_ENV_BIN):$(PATH)" \

140− SPHINX_CONF=$(patsubst books/%.live,%,$@)/conf.py \

141− $(PY_ENV_BIN)/sphinx-autobuild --poll -B --host 0.0.0.0 --port 8080 $(SPHINX_VERBOSE) $(SPHINXOPTS)\

142− -b html \

143− -c $(DOCS_FOLDER) \

144− -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \

145− $(BOOKS_FOLDER)/$(patsubst books/%.live,%,$@) \

146− $(BOOKS_DIST)/$(patsubst books/%.live,%,$@)

147−

148−$(BOOKS_PDF): %.pdf : %.latex

149− $(Q)cd $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@); make all-pdf

150− $(Q)mkdir -p $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf

151− $(Q)cp -v $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@)/*.pdf $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf

152− @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@))/pdf"

153−

154−PHONY += $(BOOKS_LATEX)

155−$(BOOKS_LATEX): pyenvinstall | $(BOOKS_DIST)

156− SPHINX_CONF=$(patsubst books/%.latex,%,$@)/conf.py \

157− $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\

158− -b latex \

159− -c $(DOCS_FOLDER) \

160− -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \

161− $(BOOKS_FOLDER)/$(patsubst books/%.latex,%,$@) \

162− $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@)

163− @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))"

164−

165−$(BOOKS_CLEAN):

166− $(Q)rm -rf $(BOOKS_DIST)/$(patsubst books/%.clean,%,$@) \

167− $(DOCS_BUILD)/books/$(patsubst books/%.clean,%,$@) \

168− $(DOCS_BUILD)/latex/$(patsubst books/%.clean,%,$@)

169−

170−# github pages

171−PHONY += prepare-gh-pages

172−prepare-gh-pages:

173− cp -r $(DOCS_DIST)/* $(GH_PAGES)/

174− touch $(GH_PAGES)/.nojekyll

175− echo "<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>" > $(GH_PAGES)/404.html

176−

177−PHONY += gh-pages

178−gh-pages: docs-clean docs

179− - git worktree remove -f $(GH_PAGES) || exit 0

180− - git branch -D gh-pages || exit 0

181− git worktree add --no-checkout $(GH_PAGES) master

182− cd $(GH_PAGES); git checkout --orphan gh-pages && git rm -rfq .

183− $(MAKE) prepare-gh-pages

184− cd $(GH_PAGES);\

185− git add --all . ;\

186− git commit -q -m "make gh-pages: from $(shell git config --get remote.origin.url)@$(shell git rev-parse HEAD)" ;\

187− git push -f origin gh-pages

188−

189−PHONY += ci-gh-pages

190−ci-gh-pages: docs-clean docs

191− rm -Rf $(GH_PAGES)

192− mkdir -p $(GH_PAGES)

193− $(MAKE) prepare-gh-pages

194−

195−PHONY += docs-clean

196−docs-clean: $(BOOKS_CLEAN)

197− $(call cmd,sphinx_clean)

198−

199−.PHONY: $(PHONY)

utils/searx.sh+1−1

@@ -481,7 +481,7 @@ pyenv_is_available() {

481create_pyenv() {

482 rst_title "Create virtualenv (python)" section

483 echo

484− if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then

484+ if [[ ! -f "${SEARX_SRC}/manage" ]]; then

485 err_msg "to create pyenv for searx, searx has to be cloned first"

486 return 42

487 fi

utils/site-python/sphinx_build_tools.pydeleted−48

@@ -1,48 +0,0 @@

1−# -*- coding: utf-8; mode: python -*-

2−"""Implement some sphinx-build tools.

3−

4−"""

5−

6−import os

7−import sys

8−from sphinx.util.pycompat import execfile_

9−

10−# ------------------------------------------------------------------------------

11−def load_sphinx_config(namespace):

12−# ------------------------------------------------------------------------------

13−

14− u"""Load an additional configuration file into *namespace*.

15−

16− The name of the configuration file is taken from the environment

17− ``SPHINX_CONF``. The external configuration file extends (or overwrites) the

18− configuration values from the origin ``conf.py``. With this you are able to

19− maintain *build themes*. To your docs/conf.py add::

20−

21− from sphinx_build_tools import load_sphinx_config

22− ...

23−

24− # Since loadConfig overwrites settings from the global namespace, it has to be

25− # the last statement in the conf.py file

26−

27− load_sphinx_config(globals())

28−

29− """

30−

31− config_file = os.environ.get("SPHINX_CONF", None)

32− if (config_file is not None

33− and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ):

34− config_file = os.path.abspath(config_file)

35−

36− if os.path.isfile(config_file):

37− sys.stdout.write(

38− "load additional sphinx-config: %s\n"

39− % config_file)

40− config = namespace.copy()

41− config['__file__'] = config_file

42− execfile_(config_file, config)

43− del config['__file__']

44− namespace.update(config)

45− else:

46− sys.stderr.write(

47− "WARNING: additional sphinx-config not found: %s\n"

48− % config_file)