3cf3152 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

utils/searx.sh: add script to install isolated searx service (WIP)

3cf3152Markus Heiser7 years ago

Makefile+1

compare with master

unifiedsplit

@@ -79,6 +79,7 @@ test: test.pep8 test.unit test.sh test.robot

79test.sh:

80 shellcheck -x utils/lib.sh

81 shellcheck -x utils/filtron.sh

82+ shellcheck -x utils/searx.sh

83test.pep8: pyenvinstall

84test.pep8: pyenvinstall

84 $(PY_ENV_ACT); ./manage.sh pep8_check

85 $(PY_ENV_ACT); ./manage.sh pep8_check

utils/filtron.sh+2−1

@@ -192,7 +192,8 @@ assert_user() {

192 rst_title "user $SERVICE_USER" section

193 echo

194 tee_stderr 1 <<EOF | bash | prefix_stdout

195−sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME --group --gecos 'Filtron' $SERVICE_USER

195+sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME \

196+ --disabled-password --group --gecos 'Filtron' $SERVICE_USER

196sudo -H usermod -a -G shadow $SERVICE_USER

197sudo -H usermod -a -G shadow $SERVICE_USER

197groups $SERVICE_USER

198groups $SERVICE_USER

198EOF

199EOF

utils/lib.sh+180−6

@@ -2,6 +2,12 @@

2# -*- coding: utf-8; mode: sh -*-

3# shellcheck disable=SC2059,SC1117,SC2162,SC2004

5+ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"

6+ADMIN_NAME="${ADMIN_NAME:-$USER}"

8+ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}"

9+ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}"

10+

5if [[ -z "${REPO_ROOT}" ]]; then

11if [[ -z "${REPO_ROOT}" ]]; then

6 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")

12 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")

7 while [ -h "${REPO_ROOT}" ] ; do

13 while [ -h "${REPO_ROOT}" ] ; do

@@ -115,7 +121,7 @@ ask_yn() {

115 esac

121 esac

116 echo

122 echo

117 while true; do

123 while true; do

118− clean_stdin

124+ clean_stdin

119 printf "$1 ${choice} "

125 printf "$1 ${choice} "

120 # shellcheck disable=SC2086

126 # shellcheck disable=SC2086

121 read -n1 $_t

127 read -n1 $_t

@@ -240,7 +246,7 @@ choose_one() {

240 fi

246 fi

241 done

247 done

242 while true; do

248 while true; do

243− clean_stdin

249+ clean_stdin

244 printf "$1 [$default] "

250 printf "$1 [$default] "

245

251

246 if (( 10 > $max )); then

252 if (( 10 > $max )); then

@@ -333,7 +339,7 @@ install_template() {

333 info_msg "install: ${template_file}"

339 info_msg "install: ${template_file}"

334 sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \

340 sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \

335 "${template_file}" "${dst}" | prefix_stdout

341 "${template_file}" "${dst}" | prefix_stdout

336− break

342+ break

337 ;;

343 ;;

338 "leave file unchanged")

344 "leave file unchanged")

339 break

345 break

@@ -343,12 +349,180 @@ install_template() {

343 echo "// exit with CTRL-D"

349 echo "// exit with CTRL-D"

344 sudo -H -u "${owner}" -i

350 sudo -H -u "${owner}" -i

345 $DIFF_CMD "${dst}" "${template_file}"

351 $DIFF_CMD "${dst}" "${template_file}"

346− if ask_yn "did you edit ${template_file} to your needs?"; then

352+ if ask_yn "did you edit ${template_file} to your needs?"; then

347− break

353+ break

348− fi

354+ fi

349 ;;

355 ;;

350 "diff files")

356 "diff files")

351 $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout

357 $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout

352 esac

358 esac

353 done

359 done

354}

360}

361+

362+

363+# uWSGI

364+# -----

365+

366+uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}"

367+

368+uWSGI_restart() {

369+

370+ # usage: uWSGI_restart()

371+

372+ info_msg "restart uWSGI service"

373+ sudo -H systemctl restart uwsgi

374+}

375+

376+uWSGI_install_app() {

377+

378+ # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ...

379+

380+ local do_eval=""

381+ local CONF

382+

383+ if [[ "$1" == "--no-eval" ]]; then

384+ no_eval=$1; shift

385+ fi

386+

387+ for CONF in "$@"; do

388+ install_template "$no_eval" "${CONF}" root root 644

389+ uWSGI_enable_app "$(basename "${CONF}")"

390+ info_msg "enabled uWSGI app: $(basename "${CONF}")"

391+ done

392+ uWSGI_restart

393+}

394+

395+uWSGI_remove_app() {

396+

397+ # usage: uWSGI_remove_app <path.ini> ...

398+

399+ local CONF

400+ for CONF in "$@"; do

401+ uWSGI_disable_app "$(basename "${CONF}")"

402+ rm -f "$CONF"

403+ info_msg "removed uWSGI app: $(basename "${CONF}")"

404+ done

405+ uWSGI_restart

406+}

407+

408+# shellcheck disable=SC2164

409+uWSGI_enable_app() {

410+

411+ # usage: uWSGI_enable_app <path.ini>

412+

413+ local CONF=$1

414+ if [[ -z $CONF ]]; then

415+ err_msg "uWSGI_enable_app missing arguments"

416+ return 42

417+ fi

418+ pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null

419+ # shellcheck disable=SC2226

420+ ln -s "../apps-available/$(basename "${CONF}")"

421+ info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"

422+ popd >/dev/null

423+}

424+

425+uWSGI_disable_app() {

426+

427+ # usage: uWSGI_disable_app <path.ini>

428+

429+ local CONF=$1

430+ if [[ -z $CONF ]]; then

431+ err_msg "uWSGI_enable_app missing arguments"

432+ return 42

433+ fi

434+

435+ rm -f "${uWSGI_SETUP}/apps-enabled/$CONF"

436+ info_msg "disabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)"

437+}

438+

439+# distro's package manager

440+# ------------------------

441+#

442+# FIXME: Arch Linux & RHEL should be added

443+#

444+

445+pkg_install() {

446+

447+ # usage: TITEL='install foobar' pkg_install foopkg barpkg

448+

449+ rst_title "${TITLE:-installation of packages}" section

450+ echo -en "\npackage(s)::\n\n $*\n" | $FMT

451+

452+ if ! ask_yn "Should packages be installed?" Yn 30; then

453+ return 42

454+ fi

455+ # shellcheck disable=SC2068

456+ apt-get install -y $@

457+ wait_key 30

458+}

459+

460+pkg_remove() {

461+

462+ # usage: TITEL='remove foobar' pkg_remove foopkg barpkg

463+

464+ rst_title "${TITLE:-remove packages}" section

465+ echo -en "\npackage(s)::\n\n $*\n" | $FMT

466+

467+ if ! ask_yn "Should packages be removed (purge)?" Yn 30; then

468+ return 42

469+ fi

470+ apt-get purge --autoremove --ignore-missing -y "$@"

471+ wait_key 30

472+}

473+

474+pkg_is_installed() {

475+

476+ # usage: pkg_is_install foopkg || pkg_install foopkg

477+

478+ dpkg -l "$1" &> /dev/null

479+ return $?

480+}

481+

482+# git tooling

483+# -----------

484+

485+# shellcheck disable=SC2164

486+git_clone() {

487+

488+ # usage:

489+ #

490+ # git_clone <url> <name> [<branch> [<user>]]

491+ # git_clone <url> <path> [<branch> [<user>]]

492+ #

493+ # First form uses $CACHE/<name> as destination folder, second form clones

494+ # into <path>. If repository is allready cloned, merge from origin and

495+ # update working tree (if needed, the caller has to stash local changes).

496+ #

497+ # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin

498+ #

499+

500+ local url="$1"

501+ local dest="$2"

502+ local branch="$3"

503+ local user="$4"

504+ local prefix=""

505+

506+ if [[ ! "${dest:0:1}" = "/" ]]; then

507+ dest="$CACHE/$dest"

508+ fi

509+

510+ [[ -z $branch ]] && branch=master

511+ [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}"

512+ [[ -z $user ]] && prefix="sudo -H -u $user"

513+

514+ if [[ -d "${dest}" ]] ; then

515+ info_msg "already cloned: $dest"

516+ pushd "${dest}" > /dev/null

517+ $prefix git checkout -b "$(basename "$branch")" --track "$branch"

518+ $prefix git pull --all

519+ popd > /dev/null

520+

521+ else

522+ info_msg "clone into: $dest"

523+ $prefix mkdir -p "$(dirname "$dest")"

524+ pushd "${dest}" > /dev/null

525+ git clone "$url" "$(basename "$dest")"

526+ popd > /dev/null

527+ fi

528+}

utils/searx.shnew file+349

@@ -0,0 +1,349 @@

1+#!/usr/bin/env bash

2+# -*- coding: utf-8; mode: sh -*-

3+# shellcheck disable=SC2119

5+# shellcheck source=utils/lib.sh

6+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"

8+# ----------------------------------------------------------------------------

9+# config

10+# ----------------------------------------------------------------------------

11+

12+SERVICE_NAME="searx"

13+SERVICE_USER="${SERVICE_NAME}"

14+# shellcheck disable=SC2034

15+SERVICE_GROUP="${SERVICE_USER}"

16+SERVICE_HOME="/home/${SERVICE_USER}"

17+

18+SEARX_GIT_URL="https://github.com/asciimoo/searx.git"

19+SEARX_GIT_BRANCH="origin/master"

20+

21+# FIXME: Arch Linux & RHEL should be added

22+

23+SEARX_APT_PACKAGES="\

24+libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \

25+ git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \

26+ libffi-dev libssl-dev"

27+

28+SEARX_VENV="${SEARX_HOME}/searx-venv"

29+SEARX_SRC="${SEARX_HOME}/searx-src"

30+SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml"

31+SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}"

32+SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini"

33+

34+# shellcheck disable=SC2034

35+CONFIG_FILES=(

36+ "${SEARX_UWSGI_APP}"

37+)

38+

39+# shellcheck disable=SC2034

40+CONFIG_BACKUP_ENCRYPTED=(

41+ "${SEARX_SETTINGS}"

42+)

43+

44+# ----------------------------------------------------------------------------

45+usage(){

46+# ----------------------------------------------------------------------------

47+

48+ # shellcheck disable=SC1117

49+ cat <<EOF

50+

51+usage:

52+

53+ $(basename "$0") shell

54+ $(basename "$0") install [all|user]

55+ $(basename "$0") update [searx]

56+ $(basename "$0") remove [all]

57+ $(basename "$0") activate [service]

58+ $(basename "$0") deactivate [service]

59+ $(basename "$0") show [service]

60+

61+shell

62+ start interactive shell from user ${SERVICE_USER}

63+install / remove all

64+ complete setup of searx service

65+update searx

66+ Update searx installation of user ${SERVICE_USER}

67+activate

68+ activate and start service daemon (systemd unit)

69+deactivate service

70+ stop and deactivate service daemon (systemd unit)

71+install user

72+ add service user '$SERVICE_USER' at $SERVICE_HOME

73+show service

74+ show service status and log

75+EOF

76+ [ ! -z ${1+x} ] && echo -e "$1"

77+}

78+

79+main(){

80+ rst_title "$SERVICE_NAME" part

81+

82+ local _usage="ERROR: unknown or missing $1 command $2"

83+

84+ case $1 in

85+ --source-only) ;;

86+ -h|--help) usage; exit 0;;

87+

88+ shell)

89+ sudo_or_exit

90+ interactive_shell

91+ ;;

92+ show)

93+ case $2 in

94+ service)

95+ sudo_or_exit

96+ show_service

97+ ;;

98+ *) usage "$_usage"; exit 42;;

99+ esac ;;

100+ install)

101+ sudo_or_exit

102+ case $2 in

103+ all) install_all ;;

104+ user) assert_user ;;

105+ *) usage "$_usage"; exit 42;;

106+ esac ;;

107+ update)

108+ sudo_or_exit

109+ case $2 in

110+ searx) update_searx;;

111+ *) usage "$_usage"; exit 42;;

112+ esac ;;

113+ remove)

114+ sudo_or_exit

115+ case $2 in

116+ all) remove_all;;

117+ user) remove_user ;;

118+ *) usage "$_usage"; exit 42;;

119+ esac ;;

120+ activate)

121+ sudo_or_exit

122+ case $2 in

123+ service) activate_service ;;

124+ *) usage "$_usage"; exit 42;;

125+ esac ;;

126+ deactivate)

127+ sudo_or_exit

128+ case $2 in

129+ service) deactivate_service ;;

130+ *) usage "$_usage"; exit 42;;

131+ esac ;;

132+ *) usage "ERROR: unknown or missing command $1"; exit 42;;

133+ esac

134+}

135+

136+_service_prefix=" |$SERVICE_USER| "

137+

138+install_all() {

139+ rst_title "Install $SERVICE_NAME (service)"

140+ pkg_install "$SEARX_APT_PACKAGES"

141+ wait_key

142+ assert_user

143+ wait_key

144+ clone_searx

145+ wait_key

146+ create_venv

147+ wait_key

148+ configure_searx

149+ wait_key

150+ test_local_searx

151+ wait_key

152+ install_searx_uwsgi

153+ wait_key

154+

155+ # ToDo ...

156+ # install_apache_site

157+ # test_public_searx

158+ # info_msg "searX --> https://${SEARX_APACHE_DOMAIN}${SEARX_APACHE_URL}"

159+

160+}

161+

162+update_searx() {

163+ rst_title "Update searx instance"

164+

165+ echo

166+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

167+cd ${SEARX_SRC}

168+cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup

169+git stash push -m "BACKUP -- 'update server' at ($(date))"

170+git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH"

171+git pull "$SEARX_GIT_BRANCH"

172+${SEARX_SRC}/manage.sh update_packages

173+EOF

174+ configure_searx

175+

176+ rst_title "${SEARX_SETTINGS}" section

177+ rstBlock 'Diff between new setting file (<) and backup (>):'

178+ echo

179+ diff "$SEARX_SETTINGS}" "${SEARX_SETTINGS}.backup"

180+

181+ local action

182+ choose_one action "What should happen to the settings file? " \

183+ "keep new configuration" \

184+ "revert to the old configuration (backup file)" \

185+ "start interactiv shell"

186+ case $action in

187+ "keep new configuration")

188+ info_msg "continue using new settings file"

189+ ;;

190+ "revert to the old configuration (backup file)")

191+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

192+cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS}

193+EOF

194+ ;;

195+ "start interactiv shell")

196+ interactive_shell

197+ ;;

198+ esac

199+ chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}"

200+

201+ # shellcheck disable=SC2016

202+ rst_para 'Diff between local modified settings (<) and $SEARX_GIT_BRANCH branch (>):'

203+ echo

204+ git_diff

205+ wait_key

206+ uWSGI_restart

207+}

208+

209+remove_all() {

210+ rst_title "De-Install $SERVICE_NAME (service)"

211+ remove_service

212+ wait_key

213+ remove_user

214+}

215+

216+assert_user() {

217+ rst_title "user $SERVICE_USER" section

218+ echo

219+ tee_stderr 1 <<EOF | bash | prefix_stdout

220+sudo -H adduser --shell /bin/bash --system --home "$SERVICE_HOME" \

221+ --disabled-password --group --gecos 'searx' $SERVICE_USER

222+sudo -H usermod -a -G shadow $SERVICE_USER

223+groups $SERVICE_USER

224+EOF

225+ #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"

226+ #export SERVICE_HOME

227+ #echo "export SERVICE_HOME=$SERVICE_HOME"

228+}

229+

230+remove_user() {

231+ rst_title "Drop $SERVICE_USER HOME" section

232+ if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then

233+ userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout

234+ else

235+ rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged."

236+ fi

237+}

238+

239+# shellcheck disable=SC2164

240+clone_searx(){

241+ rst_title "Clone searx sources" section

242+ echo

243+ git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \

244+ "$SEARX_GIT_BRANCH" "$SERVICE_USER"

245+

246+ pushd "${SEARX_SRC}" > /dev/null

247+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

248+cd "${SEARX_SRC}"

249+git config user.email "$ADMIN_EMAIL"

250+git config user.name "$ADMIN_NAME"

251+git checkout "$SEARX_GIT_BRANCH"

252+EOF

253+ popd > /dev/null

254+}

255+

256+create_venv(){

257+ rst_title "Create virtualenv (python)" section

258+

259+ rst_para "Create venv in ${SEARX_VENV} and install needed python packages."

260+ echo

261+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

262+rm -rf "${SEARX_VENV}"

263+python3 -m venv "${SEARX_VENV}"

264+. ${SEARX_VENV}/bin/activate

265+${SEARX_SRC}/manage.sh update_packages

266+EOF

267+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

268+grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \

269+ || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile

270+EOF

271+

272+}

273+

274+configure_searx(){

275+ rst_title "Configure searx" section

276+ rst_para "Setup searx config located at $SEARX_SETTINGS"

277+ echo

278+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

279+cd ${SEARX_SRC}

280+sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS"

281+sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS"

282+EOF

283+}

284+

285+test_local_searx(){

286+ rstHeading "Testing searx instance localy" section

287+ echo

288+ tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix"

289+cd ${SEARX_SRC}

290+sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS"

291+timeout 5 python3 searx/webapp.py &

292+sleep 1

293+curl --location --verbose --head --insecure http://127.0.0.1:8888/

294+sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS"

295+EOF

296+ waitKEY

297+}

298+

299+install_searx_uwsgi() {

300+ rst_title "Install searx's uWSGI app (searx.ini)" section

301+ echo

302+ uWSGI_install_app "$SEARX_UWSGI_APP"

303+}

304+

305+remove_searx_uwsgi() {

306+ rst_title "Remove searx's uWSGI app (searx.ini)" section

307+ echo

308+ uWSGI_remove_app "$SEARX_UWSGI_APP"

309+}

310+

311+activate_service () {

312+ rst_title "Activate $SERVICE_NAME (service)" section

313+ uWSGI_enable_app "$SEARX_UWSGI_APP"

314+}

315+

316+deactivate_service () {

317+ rst_title "De-Activate $SERVICE_NAME (service)" section

318+ uWSGI_disable_app "$SEARX_UWSGI_APP"

319+}

320+

321+interactive_shell(){

322+ echo "// exit with CTRL-D"

323+ sudo -H -u "${SERVICE_USER}" -i

324+}

325+

326+git_diff(){

327+ sudo -H -u "${SERVICE_USER}" -i <<EOF

328+cd ${SEARX_REPO_FOLDER}

329+git --no-pager diff

330+EOF

331+}

332+

333+show_service () {

334+ rst_title "service status & log"

335+ echo

336+ systemctl status uwsgi.service

337+ echo

338+ read -r -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log"

339+ echo

340+ while true; do

341+ trap break 2

342+ journalctl -f -u uwsgi.service

343+ done

344+ return 0

345+}

346+

347+# ----------------------------------------------------------------------------

348+main "$@"

349+# ----------------------------------------------------------------------------