89df9d9 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

utils/searx.sh: add script to install isolated searx service (WIP)

89df9d9Markus Heiser7 years ago

utils/filtron.sh+1−1

compare with master

unifiedsplit

@@ -1,5 +1,5 @@

1#!/usr/bin/env bash

2−# -*- coding: utf-8; mode: sh -*-

2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

3# shellcheck disable=SC2119

5# shellcheck source=utils/lib.sh

utils/lib.sh+30−31

@@ -1,5 +1,5 @@

1#!/usr/bin/env bash

2−# -*- coding: utf-8; mode: sh -*-

2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

3# shellcheck disable=SC2059,SC1117,SC2162,SC2004

5ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"

@@ -265,9 +265,9 @@ choose_one() {

265 _t=""

266 err_msg "invalid choice"

267 done

268+ eval "$env_name"='${list[${REPLY}]}'

268 echo

269 echo

269 clean_stdin

270 clean_stdin

270− eval "$env_name"='${list[${REPLY}]}'

271}

272

273install_template() {

@@ -375,34 +375,32 @@ uWSGI_restart() {

375

376uWSGI_install_app() {

377

378− # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ...

378+ # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini

379

380− local do_eval=""

380+ local no_eval=""

381− local CONF

381+ local CONF=""

382

383 if [[ "$1" == "--no-eval" ]]; then

384 no_eval=$1; shift

385 fi

386

387− for CONF in "$@"; do

387+ CONF=$1

388+ # shellcheck disable=SC2086

388− install_template "$no_eval" "${CONF}" root root 644

389+ install_template $no_eval "${CONF}" root root 644

389− uWSGI_enable_app "$(basename "${CONF}")"

390+ uWSGI_enable_app "$(basename "${CONF}")"

390− info_msg "enabled uWSGI app: $(basename "${CONF}")"

391− done

392 uWSGI_restart

391 uWSGI_restart

392+ info_msg "installed uWSGI app: $(basename "${CONF}")"

393}

394

395uWSGI_remove_app() {

396

397− # usage: uWSGI_remove_app <path.ini> ...

397+ # usage: uWSGI_remove_app <path.ini>

398

399− local CONF

399+ local CONF=$1

400− for CONF in "$@"; do

401− uWSGI_disable_app "$(basename "${CONF}")"

400+ uWSGI_disable_app "$(basename "${CONF}")"

402− rm -f "$CONF"

403− info_msg "removed uWSGI app: $(basename "${CONF}")"

404− done

405 uWSGI_restart

401 uWSGI_restart

402+ rm -f "$CONF"

403+ info_msg "removed uWSGI app: $(basename "${CONF}")"

406}

404}

407

405

408# shellcheck disable=SC2164

406# shellcheck disable=SC2164

@@ -416,6 +414,7 @@ uWSGI_enable_app() {

416 return 42

414 return 42

417 fi

415 fi

418 pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null

416 pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null

417+ rm -f "$(basename "${CONF}")"

419 # shellcheck disable=SC2226

418 # shellcheck disable=SC2226

420 ln -s "../apps-available/$(basename "${CONF}")"

419 ln -s "../apps-available/$(basename "${CONF}")"

421 info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"

420 info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"

@@ -454,7 +453,6 @@ pkg_install() {

454 fi

453 fi

455 # shellcheck disable=SC2068

454 # shellcheck disable=SC2068

456 apt-get install -y $@

455 apt-get install -y $@

457− wait_key 30

458}

456}

459

457

460pkg_remove() {

458pkg_remove() {

@@ -468,7 +466,6 @@ pkg_remove() {

468 return 42

466 return 42

469 fi

467 fi

470 apt-get purge --autoremove --ignore-missing -y "$@"

468 apt-get purge --autoremove --ignore-missing -y "$@"

471− wait_key 30

472}

469}

473

470

474pkg_is_installed() {

471pkg_is_installed() {

@@ -491,7 +488,7 @@ git_clone() {

491 # git_clone <url> <path> [<branch> [<user>]]

488 # git_clone <url> <path> [<branch> [<user>]]

492 #

489 #

493 # First form uses $CACHE/<name> as destination folder, second form clones

490 # First form uses $CACHE/<name> as destination folder, second form clones

494− # into <path>. If repository is allready cloned, merge from origin and

491+ # into <path>. If repository is allready cloned, pull from <branch> and

495 # update working tree (if needed, the caller has to stash local changes).

492 # update working tree (if needed, the caller has to stash local changes).

496 #

493 #

497 # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin

494 # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin

@@ -501,7 +498,8 @@ git_clone() {

501 local dest="$2"

498 local dest="$2"

502 local branch="$3"

499 local branch="$3"

503 local user="$4"

500 local user="$4"

504− local prefix=""

501+ local bash_cmd="bash"

502+ local remote="origin"

505

503

506 if [[ ! "${dest:0:1}" = "/" ]]; then

504 if [[ ! "${dest:0:1}" = "/" ]]; then

507 dest="$CACHE/$dest"

505 dest="$CACHE/$dest"

@@ -509,20 +507,21 @@ git_clone() {

509

507

510 [[ -z $branch ]] && branch=master

508 [[ -z $branch ]] && branch=master

511 [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}"

509 [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}"

512− [[ -z $user ]] && prefix="sudo -H -u $user"

510+ [[ ! -z $user ]] && bash_cmd="sudo -H -u $user -i"

513

511

514 if [[ -d "${dest}" ]] ; then

512 if [[ -d "${dest}" ]] ; then

515 info_msg "already cloned: $dest"

513 info_msg "already cloned: $dest"

516− pushd "${dest}" > /dev/null

514+ tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| "

515+cd "${dest}"

517− $prefix git checkout -b "$(basename "$branch")" --track "$branch"

516+git checkout -m -B "$branch" --track "$remote/$branch"

518− $prefix git pull --all

517+git pull --all

519− popd > /dev/null

518+EOF

520−

521 else

519 else

522 info_msg "clone into: $dest"

520 info_msg "clone into: $dest"

521+ tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| "

523− $prefix mkdir -p "$(dirname "$dest")"

522+mkdir -p "$(dirname "$dest")"

524− pushd "${dest}" > /dev/null

523+cd "$(dirname "$dest")"

525− git clone "$url" "$(basename "$dest")"

524+git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")"

526− popd > /dev/null

525+EOF

527 fi

526 fi

528}

527}

utils/searx.sh+78−32

@@ -1,5 +1,5 @@

1#!/usr/bin/env bash

2−# -*- coding: utf-8; mode: sh -*-

2+# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

3# shellcheck disable=SC2119

5# shellcheck source=utils/lib.sh

@@ -15,8 +15,11 @@ SERVICE_USER="${SERVICE_NAME}"

15SERVICE_GROUP="${SERVICE_USER}"

16SERVICE_HOME="/home/${SERVICE_USER}"

18+# shellcheck disable=SC2034

19+SEARX_URL="127.0.0.1:8888"

20+

18SEARX_GIT_URL="https://github.com/asciimoo/searx.git"

21SEARX_GIT_URL="https://github.com/asciimoo/searx.git"

19−SEARX_GIT_BRANCH="origin/master"

22+SEARX_GIT_BRANCH="master"

21# FIXME: Arch Linux & RHEL should be added

24# FIXME: Arch Linux & RHEL should be added

@@ -25,8 +28,8 @@ libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \

25 git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \

28 git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \

26 libffi-dev libssl-dev"

29 libffi-dev libssl-dev"

28−SEARX_VENV="${SEARX_HOME}/searx-venv"

31+SEARX_PYENV="${SERVICE_HOME}/searx-pyenv"

29−SEARX_SRC="${SEARX_HOME}/searx-src"

32+SEARX_SRC="${SERVICE_HOME}/searx-src"

30SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml"

33SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml"

31SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}"

34SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}"

32SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini"

35SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini"

@@ -51,25 +54,26 @@ usage(){

51usage:

54usage:

53 $(basename "$0") shell

56 $(basename "$0") shell

54− $(basename "$0") install [all|user]

57+ $(basename "$0") install [all|user|pyenv|searx-src]

55 $(basename "$0") update [searx]

58 $(basename "$0") update [searx]

56− $(basename "$0") remove [all]

59+ $(basename "$0") remove [all|user|pyenv|searx-src]

57 $(basename "$0") activate [service]

60 $(basename "$0") activate [service]

58 $(basename "$0") deactivate [service]

61 $(basename "$0") deactivate [service]

59 $(basename "$0") show [service]

62 $(basename "$0") show [service]

61shell

64shell

62 start interactive shell from user ${SERVICE_USER}

65 start interactive shell from user ${SERVICE_USER}

63−install / remove all

66+install / remove

64− complete setup of searx service

67+ all: complete (de-) installation of searx service

68+ user: add/remove service user '$SERVICE_USER' at $SERVICE_HOME

69+ searx-src: clone $SEARX_GIT_URL

70+ pyenv: create/remove virtualenv (python) in $SEARX_PYENV

65update searx

71update searx

66 Update searx installation of user ${SERVICE_USER}

72 Update searx installation of user ${SERVICE_USER}

67activate

73activate

68 activate and start service daemon (systemd unit)

74 activate and start service daemon (systemd unit)

69deactivate service

75deactivate service

70 stop and deactivate service daemon (systemd unit)

76 stop and deactivate service daemon (systemd unit)

71−install user

72− add service user '$SERVICE_USER' at $SERVICE_HOME

73show service

77show service

74 show service status and log

78 show service status and log

75EOF

79EOF

@@ -102,6 +106,8 @@ main(){

102 case $2 in

106 case $2 in

103 all) install_all ;;

107 all) install_all ;;

104 user) assert_user ;;

108 user) assert_user ;;

109+ pyenv) create_pyenv ;;

110+ searx-src) clone_searx ;;

105 *) usage "$_usage"; exit 42;;

111 *) usage "$_usage"; exit 42;;

106 esac ;;

112 esac ;;

107 update)

113 update)

@@ -115,6 +121,8 @@ main(){

115 case $2 in

121 case $2 in

116 all) remove_all;;

122 all) remove_all;;

117 user) remove_user ;;

123 user) remove_user ;;

124+ pyenv) remove_pyenv ;;

125+ searx-src) remove_searx ;;

118 *) usage "$_usage"; exit 42;;

126 *) usage "$_usage"; exit 42;;

119 esac ;;

127 esac ;;

120 activate)

128 activate)

@@ -143,7 +151,7 @@ install_all() {

143 wait_key

151 wait_key

144 clone_searx

152 clone_searx

145 wait_key

153 wait_key

146− create_venv

154+ create_pyenv

147 wait_key

155 wait_key

148 configure_searx

156 configure_searx

149 wait_key

157 wait_key

@@ -167,7 +175,7 @@ update_searx() {

167cd ${SEARX_SRC}

175cd ${SEARX_SRC}

168cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup

176cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup

169git stash push -m "BACKUP -- 'update server' at ($(date))"

177git stash push -m "BACKUP -- 'update server' at ($(date))"

170−git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH"

178+git checkout -b $SEARX_GIT_BRANCH" --track "$SEARX_GIT_BRANCH"

171git pull "$SEARX_GIT_BRANCH"

179git pull "$SEARX_GIT_BRANCH"

172${SEARX_SRC}/manage.sh update_packages

180${SEARX_SRC}/manage.sh update_packages

173EOF

181EOF

@@ -185,16 +193,16 @@ EOF

185 "start interactiv shell"

193 "start interactiv shell"

186 case $action in

194 case $action in

187 "keep new configuration")

195 "keep new configuration")

188− info_msg "continue using new settings file"

196+ info_msg "continue using new settings file"

189− ;;

197+ ;;

190 "revert to the old configuration (backup file)")

198 "revert to the old configuration (backup file)")

191 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

199 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

192cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS}

200cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS}

193EOF

201EOF

194− ;;

202+ ;;

195− "start interactiv shell")

203+ "start interactiv shell")

196− interactive_shell

204+ interactive_shell

197− ;;

205+ ;;

198 esac

206 esac

199 chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}"

207 chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}"

200

208

@@ -208,7 +216,10 @@ EOF

208

216

209remove_all() {

217remove_all() {

210 rst_title "De-Install $SERVICE_NAME (service)"

218 rst_title "De-Install $SERVICE_NAME (service)"

211− remove_service

219+ if ! ask_yn "Do you really want to deinstall $SERVICE_NAME?"; then

220+ return

221+ fi

222+ remove_searx_uwsgi

212 wait_key

223 wait_key

213 remove_user

224 remove_user

214}

225}

@@ -240,35 +251,71 @@ remove_user() {

240clone_searx(){

251clone_searx(){

241 rst_title "Clone searx sources" section

252 rst_title "Clone searx sources" section

242 echo

253 echo

254+ SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)"

255+ if [[ ! "${SERVICE_HOME}" ]]; then

256+ err_msg "to clone searx sources, user $SERVICE_USER hast to be created first"

257+ return 42

258+ fi

259+ export SERVICE_HOME

260+

243 git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \

261 git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \

244− "$SEARX_GIT_BRANCH" "$SERVICE_USER"

262+ "$SEARX_GIT_BRANCH" "$SERVICE_USER"

245

263

246 pushd "${SEARX_SRC}" > /dev/null

264 pushd "${SEARX_SRC}" > /dev/null

247 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

265 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

248cd "${SEARX_SRC}"

266cd "${SEARX_SRC}"

249git config user.email "$ADMIN_EMAIL"

267git config user.email "$ADMIN_EMAIL"

250git config user.name "$ADMIN_NAME"

268git config user.name "$ADMIN_NAME"

251−git checkout "$SEARX_GIT_BRANCH"

269+git config --list

252EOF

270EOF

253 popd > /dev/null

271 popd > /dev/null

254}

272}

255

273

256−create_venv(){

274+remove_searx() {

257− rst_title "Create virtualenv (python)" section

275+ rst_title "Drop searx sources" section

276+ if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then

277+ rm -rf "$SEARX_SRC"

278+ else

279+ rst_para "Leave searx sources unchanged."

280+ fi

281+}

258

282

259− rst_para "Create venv in ${SEARX_VENV} and install needed python packages."

283+create_pyenv(){

284+ rst_title "Create virtualenv (python)" section

260 echo

285 echo

286+ if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then

287+ err_msg "to create pyenv for searx, searx has to be cloned first"

288+ return 42

289+ fi

290+ info_msg "create pyenv in ${SEARX_PYENV}"

261 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

291 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

262−rm -rf "${SEARX_VENV}"

292+rm -rf "${SEARX_PYENV}"

263−python3 -m venv "${SEARX_VENV}"

293+python3 -m venv "${SEARX_PYENV}"

264−. ${SEARX_VENV}/bin/activate

294+grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \

265−${SEARX_SRC}/manage.sh update_packages

295+ || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile

296+EOF

297+ info_msg "inspect python's virtual environment"

298+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

299+command -v python && python --version

266EOF

300EOF

301+ wait_key

302+ info_msg "install needed python packages"

267 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

303 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

268−grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \

304+${SEARX_SRC}/manage.sh update_packages

269− || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile

270EOF

305EOF

306+}

271

307

308+remove_pyenv(){

309+ rst_title "Remove virtualenv (python)" section

310+ if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then

311+ return

312+ fi

313+ info_msg "remove pyenv activation from ~/.profile"

314+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

315+grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.##

316+mv ~/.profile.## ~/.profile

317+EOF

318+ rm -rf "${SEARX_PYENV}"

272}

319}

273

320

274configure_searx(){

321configure_searx(){

@@ -283,7 +330,7 @@ EOF

283}

330}

284

331

285test_local_searx(){

332test_local_searx(){

286− rstHeading "Testing searx instance localy" section

333+ rst_title "Testing searx instance localy" section

287 echo

334 echo

288 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

335 tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

289cd ${SEARX_SRC}

336cd ${SEARX_SRC}

@@ -293,7 +340,6 @@ sleep 1

293curl --location --verbose --head --insecure http://127.0.0.1:8888/

340curl --location --verbose --head --insecure http://127.0.0.1:8888/

294sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS"

341sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS"

295EOF

342EOF

296− waitKEY

297}

343}

298

344

299install_searx_uwsgi() {

345install_searx_uwsgi() {

utils/templates/etc/uwsgi/apps-available/searx.ininew file+62

@@ -0,0 +1,62 @@

1+[uwsgi]

3+# uWSGI core

4+# ----------

5+#

6+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core

8+# Who will run the code

9+uid = ${SERVICE_USER}

10+gid = ${SERVICE_GROUP}

11+

12+# chdir to specified directory before apps loading

13+chdir = ${SEARX_SRC}

14+

15+# disable logging for privacy

16+disable-logging = true

17+

18+# The right granted on the created socket

19+chmod-socket = 666

20+

21+# Plugin to use and interpretor config

22+single-interpreter = true

23+

24+# enable master process

25+master = true

26+

27+# load apps in each worker instead of the master

28+lazy-apps = true

29+

30+# load uWSGI plugins

31+plugin = python3

32+

33+# By default the Python plugin does not initialize the GIL. This means your

34+# app-generated threads will not run. If you need threads, remember to enable

35+# them with enable-threads. Running uWSGI in multithreading mode (with the

36+# threads options) will automatically enable threading support. This *strange*

37+# default behaviour is for performance reasons.

38+enable-threads = true

39+

40+

41+# plugin: python

42+# --------------

43+#

44+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python

45+

46+# load a WSGI module

47+module = searx.webapp

48+

49+# set PYTHONHOME/virtualenv

50+virtualenv = ${SEARX_PYENV}

51+

52+# add directory (or glob) to pythonpath

53+pythonpath = ${SERVICE_HOME}

54+

55+

56+# plugin http

57+# -----------

58+#

59+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http

60+

61+# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html

62+http = ${SEARX_URL}