utils/filtron.sh · searx

git.iko.soy/searx

on engine-data help sign in

searx/utils/filtron.sh

561 lines 16 kB history raw

1#!/usr/bin/env bash

2# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

3# SPDX-License-Identifier: AGPL-3.0-or-later

4# shellcheck disable=SC2119,SC2001

6# shellcheck source=utils/lib.sh

7source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"

8# shellcheck source=utils/brand.env

9source "${REPO_ROOT}/utils/brand.env"

10source_dot_config

11source "${REPO_ROOT}/utils/lxc-searx.env"

12in_container && lxc_set_suite_env

14# ----------------------------------------------------------------------------

15# config

16# ----------------------------------------------------------------------------

18PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}"

19PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/$[^@]*@$\?$[^:/]*$.*/\2/')}"

21FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \

22| sed -e 's,^.*://[^/]*$/.*$,\1,g')}"

23[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/

25FILTRON_ETC="/etc/filtron"

26FILTRON_RULES="$FILTRON_ETC/rules.json"

28FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"

29FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}"

30FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}"

32SERVICE_NAME="filtron"

33SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"

34SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"

35SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"

36SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"

37# shellcheck disable=SC2034

38SERVICE_GROUP="${SERVICE_USER}"

40# shellcheck disable=SC2034

41SERVICE_GROUP="${SERVICE_USER}"

43GO_ENV="${SERVICE_HOME}/.go_env"

44GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz"

45GO_TAR=$(basename "$GO_PKG_URL")

47APACHE_FILTRON_SITE="searx.conf"

48NGINX_FILTRON_SITE="searx.conf"

50# shellcheck disable=SC2034

51CONFIG_FILES=(

52 "${FILTRON_RULES}"

53 "${SERVICE_SYSTEMD_UNIT}"

54)

56# ----------------------------------------------------------------------------

57usage() {

58# ----------------------------------------------------------------------------

60 # shellcheck disable=SC1117

61 cat <<EOF

62usage::

63 $(basename "$0") shell

64 $(basename "$0") install [all|user|rules]

65 $(basename "$0") update [filtron]

66 $(basename "$0") remove [all]

67 $(basename "$0") activate [service]

68 $(basename "$0") deactivate [service]

69 $(basename "$0") inspect [service]

70 $(basename "$0") option [debug-on|debug-off]

71 $(basename "$0") apache [install|remove]

72 $(basename "$0") nginx [install|remove]

74shell

75 start interactive shell from user ${SERVICE_USER}

76install / remove

77 :all: complete setup of filtron service

78 :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)

79 :rules: reinstall filtron rules $FILTRON_RULES

80update filtron

81 Update filtron installation ($SERVICE_HOME)

82activate service

83 activate and start service daemon (systemd unit)

84deactivate service

85 stop and deactivate service daemon (systemd unit)

86inspect service

87 show service status and log

88option

89 set one of the available options

90apache (${PUBLIC_URL})

91 :install: apache site with a reverse proxy (ProxyPass)

92 :remove: apache site ${APACHE_FILTRON_SITE}

93nginx (${PUBLIC_URL})

94 :install: nginx site with a reverse proxy (ProxyPass)

95 :remove: nginx site ${NGINX_FILTRON_SITE}

97filtron rules: ${FILTRON_RULES}

99If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file::

100 PUBLIC_URL : ${PUBLIC_URL}

101 PUBLIC_HOST : ${PUBLIC_HOST}

102 SERVICE_USER : ${SERVICE_USER}

103 FILTRON_TARGET : ${FILTRON_TARGET}

104 FILTRON_API : ${FILTRON_API}

105 FILTRON_LISTEN : ${FILTRON_LISTEN}

106EOF

107 if in_container; then

108 # in containers the service is listening on 0.0.0.0 (see lxc-searx.env)

109 for ip in $(global_IPs) ; do

110 if [[ $ip =~ .*:.* ]]; then

111 echo " container URL (IPv6): http://[${ip#*|}]:4005/"

112 else

113 # IPv4:

114 echo " container URL (IPv4): http://${ip#*|}:4005/"

115 fi

116 done

117 fi

118 [[ -n ${1} ]] && err_msg "$1"

119}

120

121main() {

122 required_commands \

123 sudo install git wget curl \

124 || exit

125

126 local _usage="unknown or missing $1 command $2"

127

128 case $1 in

129 --getenv) var="$2"; echo "${!var}"; exit 0;;

130 -h|--help) usage; exit 0;;

131

132 shell)

133 sudo_or_exit

134 interactive_shell "${SERVICE_USER}"

135 ;;

136 inspect)

137 case $2 in

138 service)

139 sudo_or_exit

140 inspect_service

141 ;;

142 *) usage "$_usage"; exit 42;;

143 esac ;;

144 install)

145 rst_title "$SERVICE_NAME" part

146 sudo_or_exit

147 case $2 in

148 all) install_all ;;

149 user) assert_user ;;

150 rules)

151 rst_title "Re-Install filtron rules"

152 echo

153 install_template --no-eval "$FILTRON_RULES" root root 644

154 systemd_restart_service "${SERVICE_NAME}"

155 ;;

156 *) usage "$_usage"; exit 42;;

157 esac ;;

158 update)

159 sudo_or_exit

160 case $2 in

161 filtron) update_filtron ;;

162 *) usage "$_usage"; exit 42;;

163 esac ;;

164 remove)

165 sudo_or_exit

166 case $2 in

167 all) remove_all;;

168 user) drop_service_account "${SERVICE_USER}" ;;

169 *) usage "$_usage"; exit 42;;

170 esac ;;

171 activate)

172 sudo_or_exit

173 case $2 in

174 service) systemd_activate_service "${SERVICE_NAME}" ;;

175 *) usage "$_usage"; exit 42;;

176 esac ;;

177 deactivate)

178 sudo_or_exit

179 case $2 in

180 service) systemd_deactivate_service "${SERVICE_NAME}" ;;

181 *) usage "$_usage"; exit 42;;

182 esac ;;

183 apache)

184 sudo_or_exit

185 case $2 in

186 install) install_apache_site ;;

187 remove) remove_apache_site ;;

188 *) usage "$_usage"; exit 42;;

189 esac ;;

190 nginx)

191 sudo_or_exit

192 case $2 in

193 install) install_nginx_site ;;

194 remove) remove_nginx_site ;;

195 *) usage "$_usage"; exit 42;;

196 esac ;;

197 option)

198 sudo_or_exit

199 case $2 in

200 debug-on) echo; enable_debug ;;

201 debug-off) echo; disable_debug ;;

202 *) usage "$_usage"; exit 42;;

203 esac ;;

204 doc) rst-doc ;;

205 *) usage "unknown or missing command $1"; exit 42;;

206 esac

207}

208

209install_all() {

210 rst_title "Install $SERVICE_NAME (service)"

211 assert_user

212 wait_key

213 install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}"

214 wait_key

215 install_filtron

216 wait_key

217 systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"

218 wait_key

219 echo

220 if ! service_is_available "http://${FILTRON_LISTEN}" ; then

221 err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"

222 fi

223 if apache_is_installed; then

224 info_msg "Apache is installed on this host."

225 if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then

226 install_apache_site

227 fi

228 elif nginx_is_installed; then

229 info_msg "nginx is installed on this host."

230 if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then

231 install_nginx_site

232 fi

233 fi

234 if ask_yn "Do you want to inspect the installation?" Ny; then

235 inspect_service

236 fi

237

238}

239

240remove_all() {

241 rst_title "De-Install $SERVICE_NAME (service)"

242

243 rst_para "\

244It goes without saying that this script can only be used to remove

245installations that were installed with this script."

246

247 if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then

248 return 42

249 fi

250 drop_service_account "${SERVICE_USER}"

251 rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout

252 if service_is_available "${PUBLIC_URL}"; then

253 MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10

254 fi

255}

256

257assert_user() {

258 rst_title "user $SERVICE_USER" section

259 echo

260 tee_stderr 1 <<EOF | bash | prefix_stdout

261useradd --shell /bin/bash --system \

262 --home-dir "$SERVICE_HOME" \

263 --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER

264mkdir "$SERVICE_HOME"

265chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"

266groups $SERVICE_USER

267EOF

268 SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"

269 export SERVICE_HOME

270 echo "export SERVICE_HOME=$SERVICE_HOME"

271

272 cat > "$GO_ENV" <<EOF

273export GOPATH=\$HOME/go-apps

274export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin

275EOF

276 echo "Environment $GO_ENV has been setup."

277

278 tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"

279grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile

280EOF

281}

282

283filtron_is_installed() {

284 [[ -f $SERVICE_HOME/go-apps/bin/filtron ]]

285}

286

287_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} "

288

289install_filtron() {

290 rst_title "Install filtron in user's ~/go-apps" section

291 echo

292 tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"

293go get -v -u github.com/asciimoo/filtron

294EOF

295 install_template --no-eval "$FILTRON_RULES" root root 644

296}

297

298update_filtron() {

299 rst_title "Update filtron" section

300 echo

301 tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr"

302go get -v -u github.com/asciimoo/filtron

303EOF

304}

305

306inspect_service() {

307

308 rst_title "service status & log"

309

310 cat <<EOF

311

312sourced ${DOT_CONFIG#"$REPO_ROOT/"} :

313

314 PUBLIC_URL : ${PUBLIC_URL}

315 PUBLIC_HOST : ${PUBLIC_HOST}

316 FILTRON_URL_PATH : ${FILTRON_URL_PATH}

317 FILTRON_API : ${FILTRON_API}

318 FILTRON_LISTEN : ${FILTRON_LISTEN}

319 FILTRON_TARGET : ${FILTRON_TARGET}

320

321EOF

322

323 if service_account_is_available "$SERVICE_USER"; then

324 info_msg "service account $SERVICE_USER available."

325 else

326 err_msg "service account $SERVICE_USER not available!"

327 fi

328 if go_is_available "$SERVICE_USER"; then

329 info_msg "~$SERVICE_USER: go is installed"

330 else

331 err_msg "~$SERVICE_USER: go is not installed"

332 fi

333 if filtron_is_installed; then

334 info_msg "~$SERVICE_USER: filtron app is installed"

335 else

336 err_msg "~$SERVICE_USER: filtron app is not installed!"

337 fi

338

339 if ! service_is_available "http://${FILTRON_API}"; then

340 err_msg "API not available at: http://${FILTRON_API}"

341 fi

342

343 if ! service_is_available "http://${FILTRON_LISTEN}" ; then

344 err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}"

345 fi

346

347 if service_is_available "http://${FILTRON_TARGET}" ; then

348 info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"

349 fi

350

351 if ! service_is_available "${PUBLIC_URL}"; then

352 warn_msg "Public service at ${PUBLIC_URL} is not available!"

353 if ! in_container; then

354 warn_msg "Check if public name is correct and routed or use the public IP from above."

355 fi

356 fi

357

358 if in_container; then

359 lxc_suite_info

360 else

361 info_msg "public URL --> ${PUBLIC_URL}"

362 info_msg "internal URL --> http://${FILTRON_LISTEN}"

363 fi

364

365

366 local _debug_on

367 if ask_yn "Enable filtron debug mode?"; then

368 enable_debug

369 _debug_on=1

370 fi

371 echo

372 systemctl --no-pager -l status "${SERVICE_NAME}"

373 echo

374

375 info_msg "public URL --> ${PUBLIC_URL}"

376 # shellcheck disable=SC2059

377 printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"

378 read -r -s -n1 -t 5

379 echo

380 while true; do

381 trap break 2

382 journalctl -f -u "${SERVICE_NAME}"

383 done

384

385 if [[ $_debug_on == 1 ]]; then

386 disable_debug

387 fi

388 return 0

389}

390

391

392enable_debug() {

393 info_msg "try to enable debug mode ..."

394 python <<EOF

395import sys, json

396

397debug = {

398 u'name': u'debug request'

399 , u'filters': []

400 , u'interval': 0

401 , u'limit': 0

402 , u'actions': [{u'name': u'log'}]

403}

404

405with open('$FILTRON_RULES') as rules:

406 j = json.load(rules)

407

408pos = None

409for i in range(len(j)):

410 if j[i].get('name') == 'debug request':

411 pos = i

412 break

413if pos is not None:

414 j[pos] = debug

415else:

416 j.append(debug)

417with open('$FILTRON_RULES', 'w') as rules:

418 json.dump(j, rules, indent=2, sort_keys=True)

419

420EOF

421 systemctl restart "${SERVICE_NAME}.service"

422}

423

424disable_debug() {

425 info_msg "try to disable debug mode ..."

426 python <<EOF

427import sys, json

428with open('$FILTRON_RULES') as rules:

429 j = json.load(rules)

430

431pos = None

432for i in range(len(j)):

433 if j[i].get('name') == 'debug request':

434 pos = i

435 break

436if pos is not None:

437 del j[pos]

438 with open('$FILTRON_RULES', 'w') as rules:

439 json.dump(j, rules, indent=2, sort_keys=True)

440EOF

441 systemctl restart "${SERVICE_NAME}.service"

442}

443

444install_apache_site() {

445

446 rst_title "Install Apache site $APACHE_FILTRON_SITE"

447

448 rst_para "\

449This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"

450

451 ! apache_is_installed && info_msg "Apache is not installed."

452

453 if ! ask_yn "Do you really want to continue?" Yn; then

454 return

455 else

456 install_apache

457 fi

458

459 "${REPO_ROOT}/utils/searx.sh" install uwsgi

460

461 apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"

462

463 info_msg "testing public url .."

464 if ! service_is_available "${PUBLIC_URL}"; then

465 err_msg "Public service at ${PUBLIC_URL} is not available!"

466 fi

467}

468

469remove_apache_site() {

470

471 rst_title "Remove Apache site $APACHE_FILTRON_SITE"

472

473 rst_para "\

474This removes apache site ${APACHE_FILTRON_SITE}."

475

476 ! apache_is_installed && err_msg "Apache is not installed."

477

478 if ! ask_yn "Do you really want to continue?" Yn; then

479 return

480 fi

481

482 apache_remove_site "$APACHE_FILTRON_SITE"

483

484}

485

486install_nginx_site() {

487

488 rst_title "Install nginx site $NGINX_FILTRON_SITE"

489

490 rst_para "\

491This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"

492

493 ! nginx_is_installed && info_msg "nginx is not installed."

494

495 if ! ask_yn "Do you really want to continue?" Yn; then

496 return

497 else

498 install_nginx

499 fi

500

501 "${REPO_ROOT}/utils/searx.sh" install uwsgi

502

503 # shellcheck disable=SC2034

504 SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)

505 # shellcheck disable=SC2034

506 SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH)

507 nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"

508

509 info_msg "testing public url .."

510 if ! service_is_available "${PUBLIC_URL}"; then

511 err_msg "Public service at ${PUBLIC_URL} is not available!"

512 fi

513}

514

515remove_nginx_site() {

516

517 rst_title "Remove nginx site $NGINX_FILTRON_SITE"

518

519 rst_para "\

520This removes nginx site ${NGINX_FILTRON_SITE}."

521

522 ! nginx_is_installed && err_msg "nginx is not installed."

523

524 if ! ask_yn "Do you really want to continue?" Yn; then

525 return

526 fi

527

528 nginx_remove_site "$FILTRON_FILTRON_SITE"

530}

533rst-doc() {

535 eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\""

536

537 echo -e "\n.. START install systemd unit"

538 cat <<EOF

539.. tabs::

540

541 .. group-tab:: systemd

542

543 .. code:: bash

544

545EOF

546 eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "

547 echo -e "\n.. END install systemd unit"

548

549 # for DIST_NAME in ubuntu-20.04 arch fedora centos; do

550 # (

551 # DIST_ID=${DIST_NAME%-*}

552 # DIST_VERS=${DIST_NAME#*-}

553 # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=

554 # # ...

555 # )

556 # done

557}

558

559# ----------------------------------------------------------------------------

560main "$@"

561# ----------------------------------------------------------------------------