@@ -0,0 +1,504 @@
1+#!/usr/bin/env bash
2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3+# SPDX-License-Identifier: AGPL-3.0-or-later
4+# shellcheck disable=SC2031
5+
6+# shellcheck source=utils/lib.sh
7+source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
8+# shellcheck source=utils/brand.env
9+source "${REPO_ROOT}/utils/brand.env"
10+source_dot_config
11+
12+# config
13+
14+PY_SETUP_EXTRAS='[test]'
15+NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"
16+GECKODRIVER_VERSION="v0.30.0"
17+# SPHINXOPTS=
18+
19+# These py files are linted by test.pylint(), all other files are linted by
20+# test.pep8()
21+PYLINT_FILES=(
22+ searx/preferences.py
23+ searx/testing.py
24+ searx/engines/gigablast.py
25+ searx/engines/deviantart.py
26+ searx/engines/digg.py
27+ searx/engines/google.py
28+ searx/engines/google_news.py
29+ searx/engines/google_videos.py
30+ searx/engines/google_images.py
31+ searx/engines/mediathekviewweb.py
32+ searx/engines/meilisearch.py
33+ searx/engines/solidtorrents.py
34+ searx/engines/solr.py
35+ searx/engines/sqlite.py
36+ searx/engines/springer.py
37+ searx/engines/google_scholar.py
38+ searx/engines/yahoo_news.py
39+ searx/engines/apkmirror.py
40+ searx/engines/core.py
41+ searx_extra/update/update_external_bangs.py
42+)
43+
44+PYLINT_SEARX_DISABLE_OPTION="\
45+I,C,R,\
46+W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
47+E1136"
48+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
49+PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
50+
51+help() {
52+ cat <<EOF
53+buildenv
54+ rebuild ./utils/brand.env
55+babel.compile
56+ pybabel compile ./searx/translations
57+data.*
58+ all : update searx/languages.py and ./data/*
59+ languages : update searx/data/engines_languages.json & searx/languages.py
60+ useragents: update searx/data/useragents.json with the most recent versions of Firefox.
61+docs.*
62+ html : build HTML documentation
63+ live : autobuild HTML documentation while editing
64+ gh-pages : deploy on gh-pages branch
65+ prebuild : build reST include files (./${DOCS_BUILD}/includes)
66+ clean : clean documentation build
67+docker
68+ build : build docker image
69+ push : build and push docker image
70+gecko.driver
71+ download & install geckodriver if not already installed (required for
72+ robot_tests)
73+node.*
74+ env : download & install npm dependencies locally
75+ clean : drop npm installations
76+py.*
77+ build : Build python packages at ./${PYDIST}
78+ clean : delete virtualenv and intermediate py files
79+pyenv.* :
80+ install : developer install of searx into virtualenv
81+ uninstall : uninstall developer installation
82+ cmd ... : run command ... in virtualenv
83+ OK : test if virtualenv is OK
84+pypi.upload:
85+ Upload python packages to PyPi (to test use pypi.upload.test)
86+test.* :
87+ pylint : lint PYLINT_FILES, searx/engines, searx & tests
88+ pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
89+ unit : run unit tests
90+ coverage : run unit tests with coverage
91+ robot : run robot test
92+ clean : clean intermediate test stuff
93+themes.* :
94+ all : build all themes
95+ oscar : build oscar theme
96+ simple : build simple theme
97+EOF
98+}
99+
100+
101+if [ "$VERBOSE" = "1" ]; then
102+ SPHINX_VERBOSE="-v"
103+ PYLINT_VERBOSE="-v"
104+fi
105+
106+# needed by sphinx-docs
107+export DOCS_BUILD
108+
109+buildenv() {
110+ SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
111+ | prefix_stdout "${_Blue}BUILDENV${_creset} "
112+ return "${PIPESTATUS[0]}"
113+}
114+
115+babel.compile() {
116+ build_msg BABEL compile
117+ pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
118+ dump_return $?
119+}
120+
121+
122+data.all() {
123+ data.languages
124+ data.useragents
125+ build_msg DATA "update searx/data/ahmia_blacklist.txt"
126+ pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py
127+ build_msg DATA "update searx/data/wikidata_units.json"
128+ pyenv.cmd python searx_extra/update/update_wikidata_units.py
129+ build_msg DATA "update searx/data/currencies.json"
130+ pyenv.cmd python searx_extra/update/update_currencies.py
131+}
132+
133+
134+data.languages() {
135+ ( set -e
136+ build_msg ENGINES "fetch languages .."
137+ pyenv.cmd python searx_extra/update/update_languages.py
138+ build_msg ENGINES "update update searx/languages.py"
139+ build_msg DATA "update searx/data/engines_languages.json"
140+ )
141+ dump_return $?
142+}
143+
144+data.useragents() {
145+ build_msg DATA "update searx/data/useragents.json"
146+ pyenv.cmd python searx_extra/update/update_firefox_version.py
147+ dump_return $?
148+}
149+
150+docs.prebuild() {
151+ build_msg DOCS "build ${DOCS_BUILD}/includes"
152+ (
153+ set -e
154+ [ "$VERBOSE" = "1" ] && set -x
155+ mkdir -p "${DOCS_BUILD}/includes"
156+ ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
157+ ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
158+ ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
159+ )
160+ dump_return $?
161+}
162+
163+docker.push() {
164+ docker.build push
165+}
166+
167+# shellcheck disable=SC2119
168+docker.build() {
169+ pyenv.install
170+
171+ build_msg DOCKER build
172+ # run installation in a subprocess and activate pyenv
173+
174+ # See https://www.shellcheck.net/wiki/SC1001 and others ..
175+ # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
176+ ( set -e
177+ # shellcheck source=/dev/null
178+ source "${PY_ENV_BIN}/activate"
179+
180+ # Check if it is a git repository
181+ if [ ! -d .git ]; then
182+ die 1 "This is not Git repository"
183+ fi
184+ if [ ! -x "$(which git)" ]; then
185+ die 1 "git is not installed"
186+ fi
187+
188+ if ! git remote get-url origin 2> /dev/null; then
189+ die 1 "there is no remote origin"
190+ fi
191+
192+ # This is a git repository
193+
194+ # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
195+ # awk to remove the "v" and the "g"
196+ SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')
197+
198+ # add the suffix "-dirty" if the repository has uncommited change
199+ # /!\ HACK for searx/searx: ignore utils/brand.env
200+ git update-index -q --refresh
201+ if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
202+ SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
203+ fi
204+
205+ # Get the last git commit id, will be added to the Searx version (see Dockerfile)
206+ VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
207+ build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
208+
209+ # Check consistency between the git tag and the searx/version.py file
210+ # /! HACK : parse Python file with bash /!
211+ # otherwise it is not possible build the docker image without all Python
212+ # dependencies ( version.py loads __init__.py )
213+ # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
214+ SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
215+ if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
216+ err_msg "git tag: $SEARX_GIT_VERSION"
217+ err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
218+ die 1 "Inconsistency between the last git tag and the searx/version.py file"
219+ fi
220+
221+ # define the docker image name
222+ GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
223+ SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"
224+
225+ # build Docker image
226+ build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
227+ docker build \
228+ --build-arg GIT_URL="${GIT_URL}" \
229+ --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
230+ --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
231+ --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
232+ --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
233+ --build-arg LABEL_VCS_URL="${GIT_URL}" \
234+ --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
235+ --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
236+ -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
237+
238+ if [ "$1" = "push" ]; then
239+ docker push "${SEARX_IMAGE_NAME}:latest"
240+ docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
241+ fi
242+ )
243+ dump_return $?
244+}
245+
246+# shellcheck disable=SC2119
247+gecko.driver() {
248+ pyenv.install
249+
250+ build_msg INSTALL "gecko.driver"
251+ # run installation in a subprocess and activate pyenv
252+ ( set -e
253+ # shellcheck source=/dev/null
254+ source "${PY_ENV_BIN}/activate"
255+
256+ # TODO : check the current geckodriver version
257+ geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
258+ set +e
259+ if [ -z "$NOTFOUND" ]; then
260+ build_msg INSTALL "geckodriver already installed"
261+ return
262+ fi
263+ PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
264+ case "$PLATFORM" in
265+ "linux 32bit" | "linux2 32bit") ARCH="linux32";;
266+ "linux 64bit" | "linux2 64bit") ARCH="linux64";;
267+ "windows 32 bit") ARCH="win32";;
268+ "windows 64 bit") ARCH="win64";;
269+ "mac 64bit") ARCH="macos";;
270+ esac
271+ GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
272+
273+ build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
274+
275+ FILE="$(mktemp)"
276+ wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
277+ rm -- "$FILE"
278+ chmod 755 -- "${PY_ENV_BIN}/geckodriver"
279+ )
280+ dump_return $?
281+}
282+
283+node.env() {
284+ local err=0
285+ pushd "${REPO_ROOT}" &> /dev/null
286+ # shellcheck disable=SC2230
287+ which npm &> /dev/null || die 1 'node.env - npm is not found!'
288+
289+ ( set -e
290+ # shellcheck disable=SC2030
291+ PATH="$(npm bin):$PATH"
292+ export PATH
293+
294+ build_msg INSTALL "npm install $NPM_PACKAGES"
295+ # shellcheck disable=SC2086
296+ npm install $NPM_PACKAGES
297+
298+ cd "${REPO_ROOT}/searx/static/themes/oscar"
299+ build_msg INSTALL "($(pwd)) npm install"
300+ npm install
301+
302+ build_msg INSTALL "($(pwd)) npm install"
303+ cd "${REPO_ROOT}/searx/static/themes/simple"
304+ npm install
305+ )
306+ err=$?
307+ popd &> /dev/null
308+ dump_return "$err"
309+}
310+
311+node.clean() {
312+
313+ build_msg CLEAN "locally installed npm dependencies"
314+ rm -rf \
315+ ./node_modules \
316+ ./package-lock.json \
317+ ./searx/static/themes/oscar/package-lock.json \
318+ ./searx/static/themes/oscar/node_modules \
319+ ./searx/static/themes/simple/package-lock.json \
320+ ./searx/static/themes/simple/node_modules
321+ dump_return $?
322+}
323+
324+py.build() {
325+ build_msg BUILD "[pylint] python package ${PYDIST}"
326+ pyenv.cmd python setup.py \
327+ sdist -d "${PYDIST}" \
328+ bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
329+}
330+
331+py.clean() {
332+ build_msg CLEAN pyenv
333+ ( set -e
334+ pyenv.drop
335+ [ "$VERBOSE" = "1" ] && set -x
336+ rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
337+ find . -name '*.pyc' -exec rm -f {} +
338+ find . -name '*.pyo' -exec rm -f {} +
339+ find . -name __pycache__ -exec rm -rf {} +
340+ )
341+}
342+
343+pyenv.check() {
344+ cat <<EOF
345+import yaml
346+print('import yaml --> OK')
347+EOF
348+}
349+
350+pyenv.install() {
351+
352+ if ! pyenv.OK; then
353+ py.clean > /dev/null
354+ fi
355+ if pyenv.install.OK > /dev/null; then
356+ return 0
357+ fi
358+ ( set -e
359+ pyenv
360+ build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
361+ "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
362+ buildenv
363+ ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"
364+}
365+
366+pyenv.uninstall() {
367+ build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
368+ pyenv.cmd python setup.py develop --uninstall 2>&1 \
369+ | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
370+
371+}
372+
373+pypi.upload() {
374+ py.clean
375+ py.build
376+ # https://github.com/pypa/twine
377+ pyenv.cmd twine upload "${PYDIST}"/*
378+}
379+
380+pypi.upload.test() {
381+ py.clean
382+ py.build
383+ pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
384+}
385+
386+test.pylint() {
387+ # shellcheck disable=SC2086
388+ ( set -e
389+ build_msg TEST "[pylint] \$PYLINT_FILES"
390+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
391+ "${PYLINT_FILES[@]}"
392+
393+ build_msg TEST "[pylint] searx/engines"
394+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
395+ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
396+ --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
397+ searx/engines
398+
399+ build_msg TEST "[pylint] searx tests"
400+ pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
401+ --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
402+ --ignore=searx/engines \
403+ tests
404+ )
405+ dump_return $?
406+}
407+
408+test.pep8() {
409+ build_msg TEST 'pycodestyle (formerly pep8)'
410+ local _exclude=""
411+ printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
412+ pyenv.cmd pycodestyle \
413+ --exclude="searx/static, searx/languages.py, $_exclude " \
414+ --max-line-length=120 \
415+ --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
416+ searx tests
417+ dump_return $?
418+}
419+
420+test.unit() {
421+ build_msg TEST 'tests/unit'
422+ pyenv.cmd python -m nose2 -s tests/unit
423+ dump_return $?
424+}
425+
426+test.coverage() {
427+ build_msg TEST 'unit test coverage'
428+ ( set -e
429+ pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
430+ pyenv.cmd coverage report
431+ pyenv.cmd coverage html
432+ )
433+ dump_return $?
434+}
435+
436+test.robot() {
437+ build_msg TEST 'robot'
438+ gecko.driver
439+ PYTHONPATH=. pyenv.cmd python searx/testing.py robot
440+ dump_return $?
441+}
442+
443+test.clean() {
444+ build_msg CLEAN "test stuff"
445+ rm -rf geckodriver.log .coverage coverage/
446+ dump_return $?
447+}
448+
449+themes.all() {
450+ ( set -e
451+ node.env
452+ themes.oscar
453+ themes.simple
454+ )
455+ dump_return $?
456+}
457+
458+themes.oscar() {
459+ local gruntfile=searx/static/themes/oscar/gruntfile.js
460+ build_msg GRUNT "${gruntfile}"
461+ PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
462+ dump_return $?
463+}
464+
465+themes.simple() {
466+ local gruntfile=searx/static/themes/simple/gruntfile.js
467+ build_msg GRUNT "${gruntfile}"
468+ PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
469+ dump_return $?
470+}
471+
472+# shellcheck disable=SC2119
473+main() {
474+
475+ local _type
476+ local cmd="$1"; shift
477+
478+ if [ "$cmd" == "" ]; then
479+ help
480+ err_msg "missing command"
481+ return 42
482+ fi
483+
484+ case "$cmd" in
485+ --getenv) var="$1"; echo "${!var}";;
486+ --help) help;;
487+ --*)
488+ help
489+ err_msg "unknown option $cmd"
490+ return 42
491+ ;;
492+ *)
493+ _type="$(type -t "$cmd")"
494+ if [ "$_type" != 'function' ]; then
495+ err_msg "unknown command $1 / use --help"
496+ return 42
497+ else
498+ "$cmd" "$@"
499+ fi
500+ ;;
501+ esac
502+}
503+
504+main "$@"