@@ -1,504 +0,0 @@
1−#!/usr/bin/env bash
2−# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
3−# SPDX-License-Identifier: AGPL-3.0-or-later
4−# shellcheck disable=SC2031
5−
6−# shellcheck source=utils/lib.sh
7−source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
8−# shellcheck source=utils/brand.env
9−source "${REPO_ROOT}/utils/brand.env"
10−source_dot_config
11−
12−# config
13−
14−PY_SETUP_EXTRAS='[test]'
15−NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"
16−GECKODRIVER_VERSION="v0.30.0"
17−# SPHINXOPTS=
18−
19−# These py files are linted by test.pylint(), all other files are linted by
20−# test.pep8()
21−PYLINT_FILES=(
22− searx/preferences.py
23− searx/testing.py
24− searx/engines/gigablast.py
25− searx/engines/deviantart.py
26− searx/engines/digg.py
27− searx/engines/google.py
28− searx/engines/google_news.py
29− searx/engines/google_videos.py
30− searx/engines/google_images.py
31− searx/engines/mediathekviewweb.py
32− searx/engines/meilisearch.py
33− searx/engines/solidtorrents.py
34− searx/engines/solr.py
35− searx/engines/sqlite.py
36− searx/engines/springer.py
37− searx/engines/google_scholar.py
38− searx/engines/yahoo_news.py
39− searx/engines/apkmirror.py
40− searx/engines/core.py
41− searx_extra/update/update_external_bangs.py
42−)
43−
44−PYLINT_SEARX_DISABLE_OPTION="\
45−I,C,R,\
46−W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
47−E1136"
48−PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"
49−PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
50−
51−help() {
52− cat <<EOF
53−buildenv
54− rebuild ./utils/brand.env
55−babel.compile
56− pybabel compile ./searx/translations
57−data.*
58− all : update searx/languages.py and ./data/*
59− languages : update searx/data/engines_languages.json & searx/languages.py
60− useragents: update searx/data/useragents.json with the most recent versions of Firefox.
61−docs.*
62− html : build HTML documentation
63− live : autobuild HTML documentation while editing
64− gh-pages : deploy on gh-pages branch
65− prebuild : build reST include files (./${DOCS_BUILD}/includes)
66− clean : clean documentation build
67−docker
68− build : build docker image
69− push : build and push docker image
70−gecko.driver
71− download & install geckodriver if not already installed (required for
72− robot_tests)
73−node.*
74− env : download & install npm dependencies locally
75− clean : drop npm installations
76−py.*
77− build : Build python packages at ./${PYDIST}
78− clean : delete virtualenv and intermediate py files
79−pyenv.* :
80− install : developer install of searx into virtualenv
81− uninstall : uninstall developer installation
82− cmd ... : run command ... in virtualenv
83− OK : test if virtualenv is OK
84−pypi.upload:
85− Upload python packages to PyPi (to test use pypi.upload.test)
86−test.* :
87− pylint : lint PYLINT_FILES, searx/engines, searx & tests
88− pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
89− unit : run unit tests
90− coverage : run unit tests with coverage
91− robot : run robot test
92− clean : clean intermediate test stuff
93−themes.* :
94− all : build all themes
95− oscar : build oscar theme
96− simple : build simple theme
97−EOF
98−}
99−
100−
101−if [ "$VERBOSE" = "1" ]; then
102− SPHINX_VERBOSE="-v"
103− PYLINT_VERBOSE="-v"
104−fi
105−
106−# needed by sphinx-docs
107−export DOCS_BUILD
108−
109−buildenv() {
110− SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
111− | prefix_stdout "${_Blue}BUILDENV${_creset} "
112− return "${PIPESTATUS[0]}"
113−}
114−
115−babel.compile() {
116− build_msg BABEL compile
117− pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"
118− dump_return $?
119−}
120−
121−
122−data.all() {
123− data.languages
124− data.useragents
125− build_msg DATA "update searx/data/ahmia_blacklist.txt"
126− pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py
127− build_msg DATA "update searx/data/wikidata_units.json"
128− pyenv.cmd python searx_extra/update/update_wikidata_units.py
129− build_msg DATA "update searx/data/currencies.json"
130− pyenv.cmd python searx_extra/update/update_currencies.py
131−}
132−
133−
134−data.languages() {
135− ( set -e
136− build_msg ENGINES "fetch languages .."
137− pyenv.cmd python searx_extra/update/update_languages.py
138− build_msg ENGINES "update update searx/languages.py"
139− build_msg DATA "update searx/data/engines_languages.json"
140− )
141− dump_return $?
142−}
143−
144−data.useragents() {
145− build_msg DATA "update searx/data/useragents.json"
146− pyenv.cmd python searx_extra/update/update_firefox_version.py
147− dump_return $?
148−}
149−
150−docs.prebuild() {
151− build_msg DOCS "build ${DOCS_BUILD}/includes"
152− (
153− set -e
154− [ "$VERBOSE" = "1" ] && set -x
155− mkdir -p "${DOCS_BUILD}/includes"
156− ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
157− ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
158− ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
159− )
160− dump_return $?
161−}
162−
163−docker.push() {
164− docker.build push
165−}
166−
167−# shellcheck disable=SC2119
168−docker.build() {
169− pyenv.install
170−
171− build_msg DOCKER build
172− # run installation in a subprocess and activate pyenv
173−
174− # See https://www.shellcheck.net/wiki/SC1001 and others ..
175− # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
176− ( set -e
177− # shellcheck source=/dev/null
178− source "${PY_ENV_BIN}/activate"
179−
180− # Check if it is a git repository
181− if [ ! -d .git ]; then
182− die 1 "This is not Git repository"
183− fi
184− if [ ! -x "$(which git)" ]; then
185− die 1 "git is not installed"
186− fi
187−
188− if ! git remote get-url origin 2> /dev/null; then
189− die 1 "there is no remote origin"
190− fi
191−
192− # This is a git repository
193−
194− # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
195− # awk to remove the "v" and the "g"
196− SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')
197−
198− # add the suffix "-dirty" if the repository has uncommited change
199− # /!\ HACK for searx/searx: ignore utils/brand.env
200− git update-index -q --refresh
201− if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then
202− SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
203− fi
204−
205− # Get the last git commit id, will be added to the Searx version (see Dockerfile)
206− VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)
207− build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
208−
209− # Check consistency between the git tag and the searx/version.py file
210− # /! HACK : parse Python file with bash /!
211− # otherwise it is not possible build the docker image without all Python
212− # dependencies ( version.py loads __init__.py )
213− # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
214− SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
215− if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then
216− err_msg "git tag: $SEARX_GIT_VERSION"
217− err_msg "searx/version.py: $SEARX_PYTHON_VERSION"
218− die 1 "Inconsistency between the last git tag and the searx/version.py file"
219− fi
220−
221− # define the docker image name
222− GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
223− SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"
224−
225− # build Docker image
226− build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
227− docker build \
228− --build-arg GIT_URL="${GIT_URL}" \
229− --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
230− --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
231− --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
232− --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
233− --build-arg LABEL_VCS_URL="${GIT_URL}" \
234− --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
235− --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
236− -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .
237−
238− if [ "$1" = "push" ]; then
239− docker push "${SEARX_IMAGE_NAME}:latest"
240− docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
241− fi
242− )
243− dump_return $?
244−}
245−
246−# shellcheck disable=SC2119
247−gecko.driver() {
248− pyenv.install
249−
250− build_msg INSTALL "gecko.driver"
251− # run installation in a subprocess and activate pyenv
252− ( set -e
253− # shellcheck source=/dev/null
254− source "${PY_ENV_BIN}/activate"
255−
256− # TODO : check the current geckodriver version
257− geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
258− set +e
259− if [ -z "$NOTFOUND" ]; then
260− build_msg INSTALL "geckodriver already installed"
261− return
262− fi
263− PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
264− case "$PLATFORM" in
265− "linux 32bit" | "linux2 32bit") ARCH="linux32";;
266− "linux 64bit" | "linux2 64bit") ARCH="linux64";;
267− "windows 32 bit") ARCH="win32";;
268− "windows 64 bit") ARCH="win64";;
269− "mac 64bit") ARCH="macos";;
270− esac
271− GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
272−
273− build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
274−
275− FILE="$(mktemp)"
276− wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
277− rm -- "$FILE"
278− chmod 755 -- "${PY_ENV_BIN}/geckodriver"
279− )
280− dump_return $?
281−}
282−
283−node.env() {
284− local err=0
285− pushd "${REPO_ROOT}" &> /dev/null
286− # shellcheck disable=SC2230
287− which npm &> /dev/null || die 1 'node.env - npm is not found!'
288−
289− ( set -e
290− # shellcheck disable=SC2030
291− PATH="$(npm bin):$PATH"
292− export PATH
293−
294− build_msg INSTALL "npm install $NPM_PACKAGES"
295− # shellcheck disable=SC2086
296− npm install $NPM_PACKAGES
297−
298− cd "${REPO_ROOT}/searx/static/themes/oscar"
299− build_msg INSTALL "($(pwd)) npm install"
300− npm install
301−
302− build_msg INSTALL "($(pwd)) npm install"
303− cd "${REPO_ROOT}/searx/static/themes/simple"
304− npm install
305− )
306− err=$?
307− popd &> /dev/null
308− dump_return "$err"
309−}
310−
311−node.clean() {
312−
313− build_msg CLEAN "locally installed npm dependencies"
314− rm -rf \
315− ./node_modules \
316− ./package-lock.json \
317− ./searx/static/themes/oscar/package-lock.json \
318− ./searx/static/themes/oscar/node_modules \
319− ./searx/static/themes/simple/package-lock.json \
320− ./searx/static/themes/simple/node_modules
321− dump_return $?
322−}
323−
324−py.build() {
325− build_msg BUILD "[pylint] python package ${PYDIST}"
326− pyenv.cmd python setup.py \
327− sdist -d "${PYDIST}" \
328− bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
329−}
330−
331−py.clean() {
332− build_msg CLEAN pyenv
333− ( set -e
334− pyenv.drop
335− [ "$VERBOSE" = "1" ] && set -x
336− rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
337− find . -name '*.pyc' -exec rm -f {} +
338− find . -name '*.pyo' -exec rm -f {} +
339− find . -name __pycache__ -exec rm -rf {} +
340− )
341−}
342−
343−pyenv.check() {
344− cat <<EOF
345−import yaml
346−print('import yaml --> OK')
347−EOF
348−}
349−
350−pyenv.install() {
351−
352− if ! pyenv.OK; then
353− py.clean > /dev/null
354− fi
355− if pyenv.install.OK > /dev/null; then
356− return 0
357− fi
358− ( set -e
359− pyenv
360− build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
361− "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
362− buildenv
363− ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"
364−}
365−
366−pyenv.uninstall() {
367− build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
368− pyenv.cmd python setup.py develop --uninstall 2>&1 \
369− | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
370−
371−}
372−
373−pypi.upload() {
374− py.clean
375− py.build
376− # https://github.com/pypa/twine
377− pyenv.cmd twine upload "${PYDIST}"/*
378−}
379−
380−pypi.upload.test() {
381− py.clean
382− py.build
383− pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
384−}
385−
386−test.pylint() {
387− # shellcheck disable=SC2086
388− ( set -e
389− build_msg TEST "[pylint] \$PYLINT_FILES"
390− pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
391− "${PYLINT_FILES[@]}"
392−
393− build_msg TEST "[pylint] searx/engines"
394− pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
395− --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
396− --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
397− searx/engines
398−
399− build_msg TEST "[pylint] searx tests"
400− pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
401− --disable="${PYLINT_SEARX_DISABLE_OPTION}" \
402− --ignore=searx/engines \
403− tests
404− )
405− dump_return $?
406−}
407−
408−test.pep8() {
409− build_msg TEST 'pycodestyle (formerly pep8)'
410− local _exclude=""
411− printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
412− pyenv.cmd pycodestyle \
413− --exclude="searx/static, searx/languages.py, $_exclude " \
414− --max-line-length=120 \
415− --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
416− searx tests
417− dump_return $?
418−}
419−
420−test.unit() {
421− build_msg TEST 'tests/unit'
422− pyenv.cmd python -m nose2 -s tests/unit
423− dump_return $?
424−}
425−
426−test.coverage() {
427− build_msg TEST 'unit test coverage'
428− ( set -e
429− pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
430− pyenv.cmd coverage report
431− pyenv.cmd coverage html
432− )
433− dump_return $?
434−}
435−
436−test.robot() {
437− build_msg TEST 'robot'
438− gecko.driver
439− PYTHONPATH=. pyenv.cmd python searx/testing.py robot
440− dump_return $?
441−}
442−
443−test.clean() {
444− build_msg CLEAN "test stuff"
445− rm -rf geckodriver.log .coverage coverage/
446− dump_return $?
447−}
448−
449−themes.all() {
450− ( set -e
451− node.env
452− themes.oscar
453− themes.simple
454− )
455− dump_return $?
456−}
457−
458−themes.oscar() {
459− local gruntfile=searx/static/themes/oscar/gruntfile.js
460− build_msg GRUNT "${gruntfile}"
461− PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
462− dump_return $?
463−}
464−
465−themes.simple() {
466− local gruntfile=searx/static/themes/simple/gruntfile.js
467− build_msg GRUNT "${gruntfile}"
468− PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"
469− dump_return $?
470−}
471−
472−# shellcheck disable=SC2119
473−main() {
474−
475− local _type
476− local cmd="$1"; shift
477−
478− if [ "$cmd" == "" ]; then
479− help
480− err_msg "missing command"
481− return 42
482− fi
483−
484− case "$cmd" in
485− --getenv) var="$1"; echo "${!var}";;
486− --help) help;;
487− --*)
488− help
489− err_msg "unknown option $cmd"
490− return 42
491− ;;
492− *)
493− _type="$(type -t "$cmd")"
494− if [ "$_type" != 'function' ]; then
495− err_msg "unknown command $1 / use --help"
496− return 42
497− else
498− "$cmd" "$@"
499− fi
500− ;;
501− esac
502−}
503−
504−main "$@"