@@ -0,0 +1,501 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2031
5+
6+# shellcheck source=utils/lib.sh
7+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
8+# shellcheck source=utils/brand.env
9+source "${REPO_ROOT}/utils/brand.env"
10+source_dot_config
11+
12+# config
13+
14+PY_SETUP_EXTRAS='[test]'
15+NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"
16+GECKODRIVER_VERSION="v0.28.0"
17+# SPHINXOPTS=
18+
19+# These py files are linted by test.pylint(), all other files are linted by
20+# test.pep8()
21+PYLINT_FILES=(
22+ searx/preferences.py
23+ searx/testing.py
24+ searx/engines/gigablast.py
25+ searx/engines/deviantart.py
26+ searx/engines/digg.py
27+ searx/engines/google.py
28+ searx/engines/google_news.py
29+ searx/engines/google_videos.py
30+ searx/engines/google_images.py
31+ searx/engines/mediathekviewweb.py
32+ searx/engines/meilisearch.py
33+ searx/engines/solidtorrents.py
34+ searx/engines/solr.py
35+ searx/engines/google_scholar.py
36+ searx/engines/yahoo_news.py
37+ searx/engines/apkmirror.py
38+ searx_extra/update/update_external_bangs.py
39+)
40+
41+PYLINT_SEARX_DISABLE_OPTION="\
42+I,C,R,\
43+W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
44+E1136"
45+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
46+PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
47+
48+help() {
49+ cat <<EOF
50+buildenv
51+ rebuild ./utils/brand.env
52+babel.compile
53+ pybabel compile ./searx/translations
54+data.*
55+ all : update searx/languages.py and ./data/*
56+ languages : update searx/data/engines_languages.json & searx/languages.py
57+ useragents: update searx/data/useragents.json with the most recent versions of Firefox.
58+docs.*
59+ html : build HTML documentation
60+ live : autobuild HTML documentation while editing
61+ gh-pages : deploy on gh-pages branch
62+ prebuild : build reST include files (./${DOCS_BUILD}/includes)
63+ clean : clean documentation build
64+docker
65+ build : build docker image
66+ push : build and push docker image
67+gecko.driver
68+ download & install geckodriver if not already installed (required for
69+ robot_tests)
70+node.*
71+ env : download & install npm dependencies locally
72+ clean : drop npm installations
73+py.*
74+ build : Build python packages at ./${PYDIST}
75+ clean : delete virtualenv and intermediate py files
76+pyenv.* :
77+ install : developer install of searx into virtualenv
78+ uninstall : uninstall developer installation
79+ cmd ... : run command ... in virtualenv
80+ OK : test if virtualenv is OK
81+pypi.upload:
82+ Upload python packages to PyPi (to test use pypi.upload.test)
83+test.* :
84+ pylint : lint PYLINT_FILES, searx/engines, searx & tests
85+ pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
86+ unit : run unit tests
87+ coverage : run unit tests with coverage
88+ robot : run robot test
89+ clean : clean intermediate test stuff
90+themes.* :
91+ all : build all themes
92+ oscar : build oscar theme
93+ simple : build simple theme
94+EOF
95+}
96+
97+
98+if [ "$VERBOSE" = "1" ]; then
99+ SPHINX_VERBOSE="-v"
100+ PYLINT_VERBOSE="-v"
101+fi
102+
103+# needed by sphinx-docs
104+export DOCS_BUILD
105+
106+buildenv() {
107+ SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
108+ | prefix_stdout "${_Blue}BUILDENV${_creset} "
109+ return "${PIPESTATUS[0]}"
110+}
111+
112+babel.compile() {
113+ build_msg BABEL compile
114+ pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
115+ dump_return $?
116+}
117+
118+
119+data.all() {
120+ data.languages
121+ data.useragents
122+ build_msg DATA "update searx/data/ahmia_blacklist.txt"
123+ pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py
124+ build_msg DATA "update searx/data/wikidata_units.json"
125+ pyenv.cmd python searx_extra/update/update_wikidata_units.py
126+ build_msg DATA "update searx/data/currencies.json"
127+ pyenv.cmd python searx_extra/update/update_currencies.py
128+}
129+
130+
131+data.languages() {
132+ ( set -e
133+ build_msg ENGINES "fetch languages .."
134+ pyenv.cmd python searx_extra/update/update_languages.py
135+ build_msg ENGINES "update update searx/languages.py"
136+ build_msg DATA "update searx/data/engines_languages.json"
137+ )
138+ dump_return $?
139+}
140+
141+data.useragents() {
142+ build_msg DATA "update searx/data/useragents.json"
143+ pyenv.cmd python searx_extra/update/update_firefox_version.py
144+ dump_return $?
145+}
146+
147+docs.prebuild() {
148+ build_msg DOCS "build ${DOCS_BUILD}/includes"
149+ (
150+ set -e
151+ [ "$VERBOSE" = "1" ] && set -x
152+ mkdir -p "${DOCS_BUILD}/includes"
153+ ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
154+ ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
155+ ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
156+ )
157+ dump_return $?
158+}
159+
160+docker.push() {
161+ docker.build push
162+}
163+
164+# shellcheck disable=SC2119
165+docker.build() {
166+ pyenv.install
167+
168+ build_msg DOCKER build
169+ # run installation in a subprocess and activate pyenv
170+
171+ # See https://www.shellcheck.net/wiki/SC1001 and others ..
172+ # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
173+ ( set -e
174+ # shellcheck source=/dev/null
175+ source "${PY_ENV_BIN}/activate"
176+
177+ # Check if it is a git repository
178+ if [ ! -d .git ]; then
179+ die 1 "This is not Git repository"
180+ fi
181+ if [ ! -x "$(which git)" ]; then
182+ die 1 "git is not installed"
183+ fi
184+
185+ if ! git remote get-url origin 2> /dev/null; then
186+ die 1 "there is no remote origin"
187+ fi
188+
189+ # This is a git repository
190+
191+ # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
192+ # awk to remove the "v" and the "g"
193+ SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')
194+
195+ # add the suffix "-dirty" if the repository has uncommited change
196+ # /!\ HACK for searx/searx: ignore utils/brand.env
197+ git update-index -q --refresh
198+ if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
199+ SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
200+ fi
201+
202+ # Get the last git commit id, will be added to the Searx version (see Dockerfile)
203+ VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
204+ build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
205+
206+ # Check consistency between the git tag and the searx/version.py file
207+ # /! HACK : parse Python file with bash /!
208+ # otherwise it is not possible build the docker image without all Python
209+ # dependencies ( version.py loads __init__.py )
210+ # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
211+ SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
212+ if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
213+ err_msg "git tag: $SEARX_GIT_VERSION"
214+ err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
215+ die 1 "Inconsistency between the last git tag and the searx/version.py file"
216+ fi
217+
218+ # define the docker image name
219+ GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
220+ SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"
221+
222+ # build Docker image
223+ build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
224+ sudo docker build \
225+ --build-arg GIT_URL="${GIT_URL}" \
226+ --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
227+ --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
228+ --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
229+ --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
230+ --build-arg LABEL_VCS_URL="${GIT_URL}" \
231+ --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
232+ --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
233+ -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
234+
235+ if [ "$1" = "push" ]; then
236+ sudo docker push "${SEARX_IMAGE_NAME}:latest"
237+ sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
238+ fi
239+ )
240+ dump_return $?
241+}
242+
243+# shellcheck disable=SC2119
244+gecko.driver() {
245+ pyenv.install
246+
247+ build_msg INSTALL "gecko.driver"
248+ # run installation in a subprocess and activate pyenv
249+ ( set -e
250+ # shellcheck source=/dev/null
251+ source "${PY_ENV_BIN}/activate"
252+
253+ # TODO : check the current geckodriver version
254+ geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
255+ set +e
256+ if [ -z "$NOTFOUND" ]; then
257+ build_msg INSTALL "geckodriver already installed"
258+ return
259+ fi
260+ PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
261+ case "$PLATFORM" in
262+ "linux 32bit" | "linux2 32bit") ARCH="linux32";;
263+ "linux 64bit" | "linux2 64bit") ARCH="linux64";;
264+ "windows 32 bit") ARCH="win32";;
265+ "windows 64 bit") ARCH="win64";;
266+ "mac 64bit") ARCH="macos";;
267+ esac
268+ GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
269+
270+ build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
271+
272+ FILE="$(mktemp)"
273+ wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
274+ rm -- "$FILE"
275+ chmod 755 -- "${PY_ENV_BIN}/geckodriver"
276+ )
277+ dump_return $?
278+}
279+
280+node.env() {
281+ local err=0
282+ pushd "${REPO_ROOT}" &> /dev/null
283+ # shellcheck disable=SC2230
284+ which npm &> /dev/null || die 1 'node.env - npm is not found!'
285+
286+ ( set -e
287+ # shellcheck disable=SC2030
288+ PATH="$(npm bin):$PATH"
289+ export PATH
290+
291+ build_msg INSTALL "npm install $NPM_PACKAGES"
292+ # shellcheck disable=SC2086
293+ npm install $NPM_PACKAGES
294+
295+ cd "${REPO_ROOT}/searx/static/themes/oscar"
296+ build_msg INSTALL "($(pwd)) npm install"
297+ npm install
298+
299+ build_msg INSTALL "($(pwd)) npm install"
300+ cd "${REPO_ROOT}/searx/static/themes/simple"
301+ npm install
302+ )
303+ err=$?
304+ popd &> /dev/null
305+ dump_return "$err"
306+}
307+
308+node.clean() {
309+
310+ build_msg CLEAN "locally installed npm dependencies"
311+ rm -rf \
312+ ./node_modules \
313+ ./package-lock.json \
314+ ./searx/static/themes/oscar/package-lock.json \
315+ ./searx/static/themes/oscar/node_modules \
316+ ./searx/static/themes/simple/package-lock.json \
317+ ./searx/static/themes/simple/node_modules
318+ dump_return $?
319+}
320+
321+py.build() {
322+ build_msg BUILD "[pylint] python package ${PYDIST}"
323+ pyenv.cmd python setup.py \
324+ sdist -d "${PYDIST}" \
325+ bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
326+}
327+
328+py.clean() {
329+ build_msg CLEAN pyenv
330+ ( set -e
331+ pyenv.drop
332+ [ "$VERBOSE" = "1" ] && set -x
333+ rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
334+ find . -name '*.pyc' -exec rm -f {} +
335+ find . -name '*.pyo' -exec rm -f {} +
336+ find . -name __pycache__ -exec rm -rf {} +
337+ )
338+}
339+
340+pyenv.check() {
341+ cat <<EOF
342+import yaml
343+print('import yaml --> OK')
344+EOF
345+}
346+
347+pyenv.install() {
348+
349+ if ! pyenv.OK; then
350+ py.clean > /dev/null
351+ fi
352+ if pyenv.install.OK > /dev/null; then
353+ return 0
354+ fi
355+ ( set -e
356+ pyenv
357+ build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
358+ "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
359+ buildenv
360+ ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"
361+}
362+
363+pyenv.uninstall() {
364+ build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
365+ pyenv.cmd python setup.py develop --uninstall 2>&1 \
366+ | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
367+
368+}
369+
370+pypi.upload() {
371+ py.clean
372+ py.build
373+ # https://github.com/pypa/twine
374+ pyenv.cmd twine upload "${PYDIST}"/*
375+}
376+
377+pypi.upload.test() {
378+ py.clean
379+ py.build
380+ pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
381+}
382+
383+test.pylint() {
384+ # shellcheck disable=SC2086
385+ ( set -e
386+ build_msg TEST "[pylint] \$PYLINT_FILES"
387+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
388+ "${PYLINT_FILES[@]}"
389+
390+ build_msg TEST "[pylint] searx/engines"
391+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
392+ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
393+ --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
394+ searx/engines
395+
396+ build_msg TEST "[pylint] searx tests"
397+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
398+ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
399+ --ignore=searx/engines \
400+ searx tests
401+ )
402+ dump_return $?
403+}
404+
405+test.pep8() {
406+ build_msg TEST 'pycodestyle (formerly pep8)'
407+ local _exclude=""
408+ printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
409+ pyenv.cmd pycodestyle \
410+ --exclude="searx/static, searx/languages.py, $_exclude " \
411+ --max-line-length=120 \
412+ --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
413+ searx tests
414+ dump_return $?
415+}
416+
417+test.unit() {
418+ build_msg TEST 'tests/unit'
419+ pyenv.cmd python -m nose2 -s tests/unit
420+ dump_return $?
421+}
422+
423+test.coverage() {
424+ build_msg TEST 'unit test coverage'
425+ ( set -e
426+ pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
427+ pyenv.cmd coverage report
428+ pyenv.cmd coverage html
429+ )
430+ dump_return $?
431+}
432+
433+test.robot() {
434+ build_msg TEST 'robot'
435+ gecko.driver
436+ PYTHONPATH=. pyenv.cmd python searx/testing.py robot
437+ dump_return $?
438+}
439+
440+test.clean() {
441+ build_msg CLEAN "test stuff"
442+ rm -rf geckodriver.log .coverage coverage/
443+ dump_return $?
444+}
445+
446+themes.all() {
447+ ( set -e
448+ node.env
449+ themes.oscar
450+ themes.simple
451+ )
452+ dump_return $?
453+}
454+
455+themes.oscar() {
456+ local gruntfile=searx/static/themes/oscar/gruntfile.js
457+ build_msg GRUNT "${gruntfile}"
458+ PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
459+ dump_return $?
460+}
461+
462+themes.simple() {
463+ local gruntfile=searx/static/themes/simple/gruntfile.js
464+ build_msg GRUNT "${gruntfile}"
465+ PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
466+ dump_return $?
467+}
468+
469+# shellcheck disable=SC2119
470+main() {
471+
472+ local _type
473+ local cmd="$1"; shift
474+
475+ if [ "$cmd" == "" ]; then
476+ help
477+ err_msg "missing command"
478+ return 42
479+ fi
480+
481+ case "$cmd" in
482+ --getenv) var="$1"; echo "${!var}";;
483+ --help) help;;
484+ --*)
485+ help
486+ err_msg "unknown option $cmd"
487+ return 42
488+ ;;
489+ *)
490+ _type="$(type -t "$cmd")"
491+ if [ "$_type" != 'function' ]; then
492+ err_msg "unknown command $1 / use --help"
493+ return 42
494+ else
495+ "$cmd" "$@"
496+ fi
497+ ;;
498+ esac
499+}
500+
501+main "$@"