89df9d9 · searx · git.iko.soy

git.iko.soy/searx

utils/searx.sh: add script to install isolated searx service (WIP)

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

commit89df9d91412c2ebb8817e72c0ca9c8cfc5852f66 authorMarkus Heiser <markus.heiser@darmarit.de> dateThu 16 Jan 2020, 14:01 (7 years ago) parent3cf3152

4 files changed+171−64

compare with master

unifiedsplit

utils/filtron.sh+1−1 utils/lib.sh+30−31 utils/searx.sh+78−32 utils/templates/etc/uwsgi/apps-available/searx.ini+62

utils/filtron.sh+1−1

@@ -1,5 +1,5 @@

11#!/usr/bin/env bash

2−# -*- coding: utf-8; mode: sh -*-

2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

33# shellcheck disable=SC2119

55# shellcheck source=utils/lib.sh

utils/lib.sh+30−31

@@ -1,5 +1,5 @@

11#!/usr/bin/env bash

2−# -*- coding: utf-8; mode: sh -*-

2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

33# shellcheck disable=SC2059,SC1117,SC2162,SC2004

55ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"

@@ -265,9 +265,9 @@ choose_one() {

265265 _t=""

266266 err_msg "invalid choice"

267267 done

268+ eval "$env_name"='${list[${REPLY}]}'

268269 echo

269270 clean_stdin

270− eval "$env_name"='${list[${REPLY}]}'

271271}

272272

273273install_template() {

@@ -375,34 +375,32 @@ uWSGI_restart() {

375375

376376uWSGI_install_app() {

377377

378− # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ...

378+ # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini

379379

380− local do_eval=""

381− local CONF

380+ local no_eval=""

381+ local CONF=""

382382

383383 if [[ "$1" == "--no-eval" ]]; then

384384 no_eval=$1; shift

385385 fi

386386

387− for CONF in "$@"; do

388− install_template "$no_eval" "${CONF}" root root 644

389− uWSGI_enable_app "$(basename "${CONF}")"

390− info_msg "enabled uWSGI app: $(basename "${CONF}")"

391− done

387+ CONF=$1

388+ # shellcheck disable=SC2086

389+ install_template $no_eval "${CONF}" root root 644

390+ uWSGI_enable_app "$(basename "${CONF}")"

392391 uWSGI_restart

392+ info_msg "installed uWSGI app: $(basename "${CONF}")"

393393}

394394

395395uWSGI_remove_app() {

396396

397− # usage: uWSGI_remove_app <path.ini> ...

397+ # usage: uWSGI_remove_app <path.ini>

398398

399− local CONF

400− for CONF in "$@"; do

401− uWSGI_disable_app "$(basename "${CONF}")"

402− rm -f "$CONF"

403− info_msg "removed uWSGI app: $(basename "${CONF}")"

404− done

399+ local CONF=$1

400+ uWSGI_disable_app "$(basename "${CONF}")"

405401 uWSGI_restart

402+ rm -f "$CONF"

403+ info_msg "removed uWSGI app: $(basename "${CONF}")"

406404}

407405

408406# shellcheck disable=SC2164

@@ -416,6 +414,7 @@ uWSGI_enable_app() {

416414 return 42

417415 fi

418416 pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null

417+ rm -f "$(basename "${CONF}")"

419418 # shellcheck disable=SC2226

420419 ln -s "../apps-available/$(basename "${CONF}")"

421420 info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"

@@ -454,7 +453,6 @@ pkg_install() {

454453 fi

455454 # shellcheck disable=SC2068

456455 apt-get install -y $@

457− wait_key 30

458456}

459457

460458pkg_remove() {

@@ -468,7 +466,6 @@ pkg_remove() {

468466 return 42

469467 fi

470468 apt-get purge --autoremove --ignore-missing -y "$@"

471− wait_key 30

472469}

473470

474471pkg_is_installed() {

@@ -491,7 +488,7 @@ git_clone() {

491488 # git_clone <url> <path> [<branch> [<user>]]

492489 #

493490 # First form uses $CACHE/<name> as destination folder, second form clones

494− # into <path>. If repository is allready cloned, merge from origin and

491+ # into <path>. If repository is allready cloned, pull from <branch> and

495492 # update working tree (if needed, the caller has to stash local changes).

496493 #

497494 # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin

@@ -501,7 +498,8 @@ git_clone() {

501498 local dest="$2"

502499 local branch="$3"

503500 local user="$4"

504− local prefix=""

501+ local bash_cmd="bash"

502+ local remote="origin"

505503

506504 if [[ ! "${dest:0:1}" = "/" ]]; then

507505 dest="$CACHE/$dest"

@@ -509,20 +507,21 @@ git_clone() {

509507

510508 [[ -z $branch ]] && branch=master

511509 [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}"

512− [[ -z $user ]] && prefix="sudo -H -u $user"

510+ [[ ! -z $user ]] && bash_cmd="sudo -H -u $user -i"

513511

514512 if [[ -d "${dest}" ]] ; then

515513 info_msg "already cloned: $dest"

516− pushd "${dest}" > /dev/null

517− $prefix git checkout -b "$(basename "$branch")" --track "$branch"

518− $prefix git pull --all

519− popd > /dev/null

520−

514+ tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| "

515+cd "${dest}"

516+git checkout -m -B "$branch" --track "$remote/$branch"

517+git pull --all

518+EOF

521519 else

522520 info_msg "clone into: $dest"

523− $prefix mkdir -p "$(dirname "$dest")"

524− pushd "${dest}" > /dev/null

525− git clone "$url" "$(basename "$dest")"

526− popd > /dev/null

521+ tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| "

522+mkdir -p "$(dirname "$dest")"

523+cd "$(dirname "$dest")"

524+git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")"

525+EOF

527526 fi

528527}

utils/searx.sh+78−32

@@ -1,5 +1,5 @@

11#!/usr/bin/env bash

2−# -*- coding: utf-8; mode: sh -*-

2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

33# shellcheck disable=SC2119

55# shellcheck source=utils/lib.sh

@@ -15,8 +15,11 @@ SERVICE_USER="${SERVICE_NAME}"

1515SERVICE_GROUP="${SERVICE_USER}"

1616SERVICE_HOME="/home/${SERVICE_USER}"

1717

18+# shellcheck disable=SC2034

19+SEARX_URL="127.0.0.1:8888"

20+

1821SEARX_GIT_URL="https://github.com/asciimoo/searx.git"

19−SEARX_GIT_BRANCH="origin/master"

22+SEARX_GIT_BRANCH="master"

2023

2124# FIXME: Arch Linux & RHEL should be added

2225

@@ -25,8 +28,8 @@ libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \

2528 git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \

2629 libffi-dev libssl-dev"

2730

28−SEARX_VENV="${SEARX_HOME}/searx-venv"

29−SEARX_SRC="${SEARX_HOME}/searx-src"

31+SEARX_PYENV="${SERVICE_HOME}/searx-pyenv"

32+SEARX_SRC="${SERVICE_HOME}/searx-src"

3033SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml"

3134SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}"

3235SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini"

@@ -51,25 +54,26 @@ usage(){

5154usage:

5255

5356 $(basename "$0") shell

54− $(basename "$0") install [all|user]

57+ $(basename "$0") install [all|user|pyenv|searx-src]

5558 $(basename "$0") update [searx]

56− $(basename "$0") remove [all]

59+ $(basename "$0") remove [all|user|pyenv|searx-src]

5760 $(basename "$0") activate [service]

5861 $(basename "$0") deactivate [service]

5962 $(basename "$0") show [service]

6063

6164shell

6265 start interactive shell from user ${SERVICE_USER}

63−install / remove all

64− complete setup of searx service

66+install / remove

67+ all: complete (de-) installation of searx service

68+ user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME

69+ searx-src: clone $SEARX_GIT_URL

70+ pyenv: create/remove virtualenv (python) in $SEARX_PYENV

6571update searx

6672 Update searx installation of user ${SERVICE_USER}

6773activate

6874 activate and start service daemon (systemd unit)

6975deactivate service

7076 stop and deactivate service daemon (systemd unit)

71−install user

72− add service user '$SERVICE_USER' at $SERVICE_HOME

7377show service

7478 show service status and log

7579EOF

@@ -102,6 +106,8 @@ main(){

102106 case $2 in

103107 all) install_all ;;

104108 user) assert_user ;;

109+ pyenv) create_pyenv ;;

110+ searx-src) clone_searx ;;

105111 *) usage "$_usage"; exit 42;;

106112 esac ;;

107113 update)

@@ -115,6 +121,8 @@ main(){

115121 case $2 in

116122 all) remove_all;;

117123 user) remove_user ;;

124+ pyenv) remove_pyenv ;;

125+ searx-src) remove_searx ;;

118126 *) usage "$_usage"; exit 42;;

119127 esac ;;

120128 activate)

@@ -143,7 +151,7 @@ install_all() {

143151 wait_key

144152 clone_searx

145153 wait_key

146− create_venv

154+ create_pyenv

147155 wait_key

148156 configure_searx

149157 wait_key

@@ -167,7 +175,7 @@ update_searx() {

167175cd ${SEARX_SRC}

168176cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup

169177git stash push -m "BACKUP -- 'update server' at ($(date))"

170−git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH"

178+git checkout -b $SEARX_GIT_BRANCH" --track "$SEARX_GIT_BRANCH"

171179git pull "$SEARX_GIT_BRANCH"

172180${SEARX_SRC}/manage.sh update_packages

173181EOF

@@ -185,16 +193,16 @@ EOF

185193 "start interactiv shell"

186194 case $action in

187195 "keep new configuration")

188− info_msg "continue using new settings file"

189− ;;

196+ info_msg "continue using new settings file"

197+ ;;

190198 "revert to the old configuration (backup file)")

191199 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

192200cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS}

193201EOF

194− ;;

195− "start interactiv shell")

196− interactive_shell

197− ;;

202+ ;;

203+ "start interactiv shell")

204+ interactive_shell

205+ ;;

198206 esac

199207 chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}"

200208

@@ -208,7 +216,10 @@ EOF

208216

209217remove_all() {

210218 rst_title "De-Install $SERVICE_NAME (service)"

211− remove_service

219+ if ! ask_yn "Do you really want to deinstall $SERVICE_NAME?"; then

220+ return

221+ fi

222+ remove_searx_uwsgi

212223 wait_key

213224 remove_user

214225}

@@ -240,35 +251,71 @@ remove_user() {

240251clone_searx(){

241252 rst_title "Clone searx sources" section

242253 echo

254+ SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)"

255+ if [[ ! "${SERVICE_HOME}" ]]; then

256+ err_msg "to clone searx sources, user $SERVICE_USER hast to be created first"

257+ return 42

258+ fi

259+ export SERVICE_HOME

260+

243261 git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \

244− "$SEARX_GIT_BRANCH" "$SERVICE_USER"

262+ "$SEARX_GIT_BRANCH" "$SERVICE_USER"

245263

246264 pushd "${SEARX_SRC}" > /dev/null

247265 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

248266cd "${SEARX_SRC}"

249267git config user.email "$ADMIN_EMAIL"

250268git config user.name "$ADMIN_NAME"

251−git checkout "$SEARX_GIT_BRANCH"

269+git config --list

252270EOF

253271 popd > /dev/null

254272}

255273

256−create_venv(){

257− rst_title "Create virtualenv (python)" section

274+remove_searx() {

275+ rst_title "Drop searx sources" section

276+ if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then

277+ rm -rf "$SEARX_SRC"

278+ else

279+ rst_para "Leave searx sources unchanged."

280+ fi

281+}

258282

259− rst_para "Create venv in ${SEARX_VENV} and install needed python packages."

283+create_pyenv(){

284+ rst_title "Create virtualenv (python)" section

260285 echo

286+ if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then

287+ err_msg "to create pyenv for searx, searx has to be cloned first"

288+ return 42

289+ fi

290+ info_msg "create pyenv in ${SEARX_PYENV}"

261291 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

262−rm -rf "${SEARX_VENV}"

263−python3 -m venv "${SEARX_VENV}"

264−. ${SEARX_VENV}/bin/activate

265−${SEARX_SRC}/manage.sh update_packages

292+rm -rf "${SEARX_PYENV}"

293+python3 -m venv "${SEARX_PYENV}"

294+grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \

295+ || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile

296+EOF

297+ info_msg "inspect python's virtual environment"

298+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

299+command -v python && python --version

266300EOF

301+ wait_key

302+ info_msg "install needed python packages"

267303 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

268−grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \

269− || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile

304+${SEARX_SRC}/manage.sh update_packages

270305EOF

306+}

271307

308+remove_pyenv(){

309+ rst_title "Remove virtualenv (python)" section

310+ if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then

311+ return

312+ fi

313+ info_msg "remove pyenv activation from ~/.profile"

314+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

315+grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.##

316+mv ~/.profile.## ~/.profile

317+EOF

318+ rm -rf "${SEARX_PYENV}"

272319}

273320

274321configure_searx(){

@@ -283,7 +330,7 @@ EOF

283330}

284331

285332test_local_searx(){

286− rstHeading "Testing searx instance localy" section

333+ rst_title "Testing searx instance localy" section

287334 echo

288335 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

289336cd ${SEARX_SRC}

@@ -293,7 +340,6 @@ sleep 1

293340curl --location --verbose --head --insecure http://127.0.0.1:8888/

294341sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS"

295342EOF

296− waitKEY

297343}

298344

299345install_searx_uwsgi() {

utils/templates/etc/uwsgi/apps-available/searx.ininew file+62

@@ -0,0 +1,62 @@

1+[uwsgi]

3+# uWSGI core

4+# ----------

5+#

6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core

8+# Who will run the code

9+uid = ${SERVICE_USER}

10+gid = ${SERVICE_GROUP}

11+

12+# chdir to specified directory before apps loading

13+chdir = ${SEARX_SRC}

14+

15+# disable logging for privacy

16+disable-logging = true

17+

18+# The right granted on the created socket

19+chmod-socket = 666

20+

21+# Plugin to use and interpretor config

22+single-interpreter = true

23+

24+# enable master process

25+master = true

26+

27+# load apps in each worker instead of the master

28+lazy-apps = true

29+

30+# load uWSGI plugins

31+plugin = python3

32+

33+# By default the Python plugin does not initialize the GIL. This means your

34+# app-generated threads will not run. If you need threads, remember to enable

35+# them with enable-threads. Running uWSGI in multithreading mode (with the

36+# threads options) will automatically enable threading support. This *strange*

37+# default behaviour is for performance reasons.

38+enable-threads = true

39+

40+

41+# plugin: python

42+# --------------

43+#

44+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python

45+

46+# load a WSGI module

47+module = searx.webapp

48+

49+# set PYTHONHOME/virtualenv

50+virtualenv = ${SEARX_PYENV}

51+

52+# add directory (or glob) to pythonpath

53+pythonpath = ${SERVICE_HOME}

54+

55+

56+# plugin http

57+# -----------

58+#

59+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http

60+

61+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html

62+http = ${SEARX_URL}