@@ -0,0 +1,506 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2031
5+
6+# shellcheck source=utils/lib.sh
7+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
8+# shellcheck source=utils/brand.env
9+source "${REPO_ROOT}/utils/brand.env"
10+source_dot_config
11+
12+# config
13+
14+PY_SETUP_EXTRAS='[test]'
15+NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"
16+GECKODRIVER_VERSION="v0.28.0"
17+# SPHINXOPTS=
18+
19+# These py files are linted by test.pylint(), all other files are linted by
20+# test.pep8()
21+PYLINT_FILES=(
22+ searx/preferences.py
23+ searx/testing.py
24+ searx/engines/gigablast.py
25+ searx/engines/deviantart.py
26+ searx/engines/digg.py
27+ searx/engines/google.py
28+ searx/engines/google_news.py
29+ searx/engines/google_videos.py
30+ searx/engines/google_images.py
31+ searx/engines/mediathekviewweb.py
32+)
33+
34+PYLINT_SEARX_DISABLE_OPTION="\
35+I,C,R,\
36+W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
37+E1136"
38+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
39+PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
40+
41+help() {
42+ cat <<EOF
43+buildenv
44+ rebuild ./utils/brand.env
45+babel.compile
46+ pybabel compile ./searx/translations
47+data.*
48+ all : update searx/languages.py and ./data/*
49+ languages : update searx/data/engines_languages.json & searx/languages.py
50+ useragents: update searx/data/useragents.json with the most recent versions of Firefox.
51+docs.*
52+ html : build HTML documentation
53+ live : autobuild HTML documentation while editing
54+ gh-pages : deploy on gh-pages branch
55+ prebuild : build reST include files (./${DOCS_BUILD}/includes)
56+ clean : clean documentation build
57+docker
58+ build : build docker image
59+ push : build and push docker image
60+gecko.driver
61+ download & install geckodriver if not already installed (required for
62+ robot_tests)
63+node.*
64+ env : download & install npm dependencies locally
65+ clean : drop npm installations
66+py.*
67+ build : Build python packages at ./${PYDIST}
68+ clean : delete virtualenv and intermediate py files
69+pyenv.* :
70+ install : developer install of searx into virtualenv
71+ uninstall : uninstall developer installation
72+ cmd ... : run command ... in virtualenv
73+ OK : test if virtualenv is OK
74+pypi.upload:
75+ Upload python packages to PyPi (to test use pypi.upload.test)
76+test.* :
77+ pylint : lint PYLINT_FILES, searx/engines, searx & tests
78+ pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
79+ unit : run unit tests
80+ coverage : run unit tests with coverage
81+ robot : run robot test
82+ clean : clean intermediate test stuff
83+themes.* :
84+ all : build all themes
85+ oscar : build oscar theme
86+ simple : build simple theme
87+ bootstrap : less compile bootstrap.min.css CSS
88+EOF
89+}
90+
91+
92+if [ "$VERBOSE" = "1" ]; then
93+ SPHINX_VERBOSE="-v"
94+ PYLINT_VERBOSE="-v"
95+fi
96+
97+# needed by sphinx-docs
98+export DOCS_BUILD
99+
100+buildenv() {
101+ SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
102+ | prefix_stdout "${_Blue}BUILDENV${_creset} "
103+ return "${PIPESTATUS[0]}"
104+}
105+
106+babel.compile() {
107+ build_msg BABEL compile
108+ pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
109+ dump_return $?
110+}
111+
112+
113+data.all() {
114+ data.languages
115+ data.useragents
116+ build_msg DATA "update searx/data/ahmia_blacklist.txt"
117+ pyenv.cmd python utils/fetch_ahmia_blacklist.py
118+ build_msg DATA "update searx/data/wikidata_units.json"
119+ pyenv.cmd python utils/fetch_wikidata_units.py
120+ build_msg DATA "update searx/data/currencies.json"
121+ pyenv.cmd python utils/fetch_currencies.py
122+}
123+
124+
125+data.languages() {
126+ ( set -e
127+ build_msg ENGINES "fetch languages .."
128+ pyenv.cmd python utils/fetch_languages.py
129+ build_msg ENGINES "update update searx/languages.py"
130+ build_msg DATA "update searx/data/engines_languages.json"
131+ )
132+ dump_return $?
133+}
134+
135+data.useragents() {
136+ build_msg DATA "update searx/data/useragents.json"
137+ pyenv.cmd python utils/fetch_firefox_version.py
138+ dump_return $?
139+}
140+
141+docs.prebuild() {
142+ build_msg DOCS "build ${DOCS_BUILD}/includes"
143+ (
144+ set -e
145+ [ "$VERBOSE" = "1" ] && set -x
146+ mkdir -p "${DOCS_BUILD}/includes"
147+ ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
148+ ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
149+ ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
150+ )
151+ dump_return $?
152+}
153+
154+docker.push() {
155+ docker.build push
156+}
157+
158+# shellcheck disable=SC2119
159+docker.build() {
160+ pyenv.install
161+
162+ build_msg DOCKER build
163+ # run installation in a subprocess and activate pyenv
164+
165+ # See https://www.shellcheck.net/wiki/SC1001 and others ..
166+ # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
167+ ( set -e
168+ # shellcheck source=/dev/null
169+ source "${PY_ENV_BIN}/activate"
170+
171+ # Check if it is a git repository
172+ if [ ! -d .git ]; then
173+ die 1 "This is not Git repository"
174+ fi
175+ if [ ! -x "$(which git)" ]; then
176+ die 1 "git is not installed"
177+ fi
178+
179+ if ! git remote get-url origin 2> /dev/null; then
180+ die 1 "there is no remote origin"
181+ fi
182+
183+ # This is a git repository
184+
185+ # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
186+ # awk to remove the "v" and the "g"
187+ SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')
188+
189+ # add the suffix "-dirty" if the repository has uncommited change
190+ # /!\ HACK for searx/searx: ignore utils/brand.env
191+ git update-index -q --refresh
192+ if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
193+ SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
194+ fi
195+
196+ # Get the last git commit id, will be added to the Searx version (see Dockerfile)
197+ VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
198+ build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
199+
200+ # Check consistency between the git tag and the searx/version.py file
201+ # /! HACK : parse Python file with bash /!
202+ # otherwise it is not possible build the docker image without all Python
203+ # dependencies ( version.py loads __init__.py )
204+ # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
205+ SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
206+ if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
207+ err_msg "git tag: $SEARX_GIT_VERSION"
208+ err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
209+ die 1 "Inconsistency between the last git tag and the searx/version.py file"
210+ fi
211+
212+ # define the docker image name
213+ GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
214+ SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"
215+
216+ # build Docker image
217+ build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
218+ sudo docker build \
219+ --build-arg GIT_URL="${GIT_URL}" \
220+ --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
221+ --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
222+ --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
223+ --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
224+ --build-arg LABEL_VCS_URL="${GIT_URL}" \
225+ --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
226+ --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
227+ -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
228+
229+ if [ "$1" = "push" ]; then
230+ sudo docker push "${SEARX_IMAGE_NAME}:latest"
231+ sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
232+ fi
233+ )
234+ dump_return $?
235+}
236+
237+# shellcheck disable=SC2119
238+gecko.driver() {
239+ pyenv.install
240+
241+ build_msg INSTALL "gecko.driver"
242+ # run installation in a subprocess and activate pyenv
243+ ( set -e
244+ # shellcheck source=/dev/null
245+ source "${PY_ENV_BIN}/activate"
246+
247+ # TODO : check the current geckodriver version
248+ geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
249+ set +e
250+ if [ -z "$NOTFOUND" ]; then
251+ build_msg INSTALL "geckodriver already installed"
252+ return
253+ fi
254+ PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
255+ case "$PLATFORM" in
256+ "linux 32bit" | "linux2 32bit") ARCH="linux32";;
257+ "linux 64bit" | "linux2 64bit") ARCH="linux64";;
258+ "windows 32 bit") ARCH="win32";;
259+ "windows 64 bit") ARCH="win64";;
260+ "mac 64bit") ARCH="macos";;
261+ esac
262+ GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
263+
264+ build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
265+
266+ FILE="$(mktemp)"
267+ wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
268+ rm -- "$FILE"
269+ chmod 755 -- "${PY_ENV_BIN}/geckodriver"
270+ )
271+ dump_return $?
272+}
273+
274+node.env() {
275+ local err=0
276+ pushd "${REPO_ROOT}" &> /dev/null
277+ # shellcheck disable=SC2230
278+ which npm &> /dev/null || die 1 'node.env - npm is not found!'
279+
280+ ( set -e
281+ # shellcheck disable=SC2030
282+ PATH="$(npm bin):$PATH"
283+ export PATH
284+
285+ build_msg INSTALL "npm install $NPM_PACKAGES"
286+ # shellcheck disable=SC2086
287+ npm install $NPM_PACKAGES
288+
289+ cd "${REPO_ROOT}/searx/static/themes/oscar"
290+ build_msg INSTALL "($(pwd)) npm install"
291+ npm install
292+
293+ build_msg INSTALL "($(pwd)) npm install"
294+ cd "${REPO_ROOT}/searx/static/themes/simple"
295+ npm install
296+ )
297+ err=$?
298+ popd &> /dev/null
299+ dump_return "$err"
300+}
301+
302+node.clean() {
303+
304+ build_msg CLEAN "locally installed npm dependencies"
305+ rm -rf \
306+ ./node_modules \
307+ ./package-lock.json \
308+ ./searx/static/themes/oscar/package-lock.json \
309+ ./searx/static/themes/oscar/node_modules \
310+ ./searx/static/themes/simple/package-lock.json \
311+ ./searx/static/themes/simple/node_modules
312+ dump_return $?
313+}
314+
315+py.build() {
316+ build_msg BUILD "[pylint] python package ${PYDIST}"
317+ pyenv.cmd python setup.py \
318+ sdist -d "${PYDIST}" \
319+ bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
320+}
321+
322+py.clean() {
323+ build_msg CLEAN pyenv
324+ ( set -e
325+ pyenv.drop
326+ [ "$VERBOSE" = "1" ] && set -x
327+ rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
328+ find . -name '*.pyc' -exec rm -f {} +
329+ find . -name '*.pyo' -exec rm -f {} +
330+ find . -name __pycache__ -exec rm -rf {} +
331+ )
332+}
333+
334+pyenv.check() {
335+ cat <<EOF
336+import yaml
337+print('import yaml --> OK')
338+EOF
339+}
340+
341+pyenv.install() {
342+
343+ if ! pyenv.OK; then
344+ py.clean > /dev/null
345+ fi
346+ if pyenv.install.OK > /dev/null; then
347+ return 0
348+ fi
349+ ( set -e
350+ py.clean > /dev/null
351+ pyenv
352+ build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
353+ "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
354+ buildenv
355+ ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"
356+}
357+
358+pyenv.uninstall() {
359+ build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
360+ pyenv.cmd python setup.py develop --uninstall 2>&1 \
361+ | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
362+
363+}
364+
365+pypi.upload() {
366+ py.clean
367+ py.build
368+ # https://github.com/pypa/twine
369+ pyenv.cmd twine upload "${PYDIST}"/*
370+}
371+
372+pypi.upload.test() {
373+ py.clean
374+ py.build
375+ pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
376+}
377+
378+test.pylint() {
379+ # shellcheck disable=SC2086
380+ ( set -e
381+ build_msg TEST "[pylint] \$PYLINT_FILES"
382+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
383+ "${PYLINT_FILES[@]}"
384+
385+ build_msg TEST "[pylint] searx/engines"
386+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
387+ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
388+ --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
389+ searx/engines
390+
391+ build_msg TEST "[pylint] searx tests"
392+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
393+ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
394+ --ignore=searx/engines \
395+ searx tests
396+ )
397+ dump_return $?
398+}
399+
400+test.pep8() {
401+ build_msg TEST 'pycodestyle (formerly pep8)'
402+ local _exclude=""
403+ printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
404+ pyenv.cmd pycodestyle \
405+ --exclude="searx/static, searx/languages.py, $_exclude " \
406+ --max-line-length=120 \
407+ --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
408+ searx tests
409+ dump_return $?
410+}
411+
412+test.unit() {
413+ build_msg TEST 'tests/unit'
414+ pyenv.cmd python -m nose2 -s tests/unit
415+ dump_return $?
416+}
417+
418+test.coverage() {
419+ build_msg TEST 'unit test coverage'
420+ ( set -e
421+ pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
422+ pyenv.cmd coverage report
423+ pyenv.cmd coverage html
424+ )
425+ dump_return $?
426+}
427+
428+test.robot() {
429+ build_msg TEST 'robot'
430+ gecko.driver
431+ PYTHONPATH=. pyenv.cmd python searx/testing.py robot
432+ dump_return $?
433+}
434+
435+test.clean() {
436+ build_msg CLEAN "test stuff"
437+ rm -rf geckodriver.log .coverage coverage/
438+ dump_return $?
439+}
440+
441+themes.all() {
442+ ( set -e
443+ node.env
444+ themes.oscar
445+ themes.simple
446+ themes.bootstrap
447+ )
448+ dump_return $?
449+}
450+
451+themes.oscar() {
452+ local gruntfile=searx/static/themes/oscar/gruntfile.js
453+ build_msg GRUNT "${gruntfile}"
454+ PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
455+ dump_return $?
456+}
457+
458+themes.simple() {
459+ local gruntfile=searx/static/themes/simple/gruntfile.js
460+ build_msg GRUNT "${gruntfile}"
461+ PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
462+ dump_return $?
463+}
464+
465+themes.bootstrap() {
466+ build_msg LESSC "searx/static/css/bootstrap.min.css"
467+ PATH="$(npm bin):$PATH" lessc \
468+ --clean-css="--s1 --advanced --compatibility=ie9" \
469+ "searx/static/less/bootstrap/bootstrap.less" \
470+ "searx/static/css/bootstrap.min.css"
471+ dump_return $?
472+}
473+
474+# shellcheck disable=SC2119
475+main() {
476+
477+ local _type
478+ local cmd="$1"; shift
479+
480+ if [ "$cmd" == "" ]; then
481+ help
482+ err_msg "missing command"
483+ return 42
484+ fi
485+
486+ case "$cmd" in
487+ --getenv) var="$1"; echo "${!var}";;
488+ --help) help;;
489+ --*)
490+ help
491+ err_msg "unknown option $cmd"
492+ return 42
493+ ;;
494+ *)
495+ _type="$(type -t "$cmd")"
496+ if [ "$_type" != 'function' ]; then
497+ err_msg "unknown command $1 / use --help"
498+ return 42
499+ else
500+ "$cmd" "$@"
501+ fi
502+ ;;
503+ esac
504+}
505+
506+main "$@"