manage · searx · git.iko.soy

git.iko.soy/searx

on update_data_update_ahmia_blacklist.py help sign in

searx/manage

504 lines 15 kB history raw

1#!/usr/bin/env bash

2# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-

3# SPDX-License-Identifier: AGPL-3.0-or-later

4# shellcheck disable=SC2031

6# shellcheck source=utils/lib.sh

7source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"

8# shellcheck source=utils/brand.env

9source "${REPO_ROOT}/utils/brand.env"

10source_dot_config

12# config

14PY_SETUP_EXTRAS='[test]'

15NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli"

16GECKODRIVER_VERSION="v0.28.0"

17# SPHINXOPTS=

19# These py files are linted by test.pylint(), all other files are linted by

20# test.pep8()

21PYLINT_FILES=(

22 searx/preferences.py

23 searx/testing.py

24 searx/engines/gigablast.py

25 searx/engines/deviantart.py

26 searx/engines/digg.py

27 searx/engines/google.py

28 searx/engines/google_news.py

29 searx/engines/google_videos.py

30 searx/engines/google_images.py

31 searx/engines/mediathekviewweb.py

32 searx/engines/meilisearch.py

33 searx/engines/solidtorrents.py

34 searx/engines/solr.py

35 searx/engines/sqlite.py

36 searx/engines/springer.py

37 searx/engines/google_scholar.py

38 searx/engines/yahoo_news.py

39 searx/engines/apkmirror.py

40 searx/engines/core.py

41 searx_extra/update/update_external_bangs.py

42)

44PYLINT_SEARX_DISABLE_OPTION="\

45I,C,R,\

46W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\

47E1136"

48PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases"

49PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"

51help() {

52 cat <<EOF

53buildenv

54 rebuild ./utils/brand.env

55babel.compile

56 pybabel compile ./searx/translations

57data.*

58 all : update searx/languages.py and ./data/*

59 languages : update searx/data/engines_languages.json & searx/languages.py

60 useragents: update searx/data/useragents.json with the most recent versions of Firefox.

61docs.*

62 html : build HTML documentation

63 live : autobuild HTML documentation while editing

64 gh-pages : deploy on gh-pages branch

65 prebuild : build reST include files (./${DOCS_BUILD}/includes)

66 clean : clean documentation build

67docker

68 build : build docker image

69 push : build and push docker image

70gecko.driver

71 download & install geckodriver if not already installed (required for

72 robot_tests)

73node.*

74 env : download & install npm dependencies locally

75 clean : drop npm installations

76py.*

77 build : Build python packages at ./${PYDIST}

78 clean : delete virtualenv and intermediate py files

79pyenv.* :

80 install : developer install of searx into virtualenv

81 uninstall : uninstall developer installation

82 cmd ... : run command ... in virtualenv

83 OK : test if virtualenv is OK

84pypi.upload:

85 Upload python packages to PyPi (to test use pypi.upload.test)

86test.* :

87 pylint : lint PYLINT_FILES, searx/engines, searx & tests

88 pep8 : pycodestyle (pep8) for all files except PYLINT_FILES

89 unit : run unit tests

90 coverage : run unit tests with coverage

91 robot : run robot test

92 clean : clean intermediate test stuff

93themes.* :

94 all : build all themes

95 oscar : build oscar theme

96 simple : build simple theme

97EOF

98}

100

101if [ "$VERBOSE" = "1" ]; then

102 SPHINX_VERBOSE="-v"

103 PYLINT_VERBOSE="-v"

104fi

105

106# needed by sphinx-docs

107export DOCS_BUILD

108

109buildenv() {

110 SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \

111 | prefix_stdout "${_Blue}BUILDENV${_creset} "

112 return "${PIPESTATUS[0]}"

113}

114

115babel.compile() {

116 build_msg BABEL compile

117 pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations"

118 dump_return $?

119}

120

121

122data.all() {

123 data.languages

124 data.useragents

125 build_msg DATA "update searx/data/ahmia_blacklist.txt"

126 pyenv.cmd python searx_extra/update/update_ahmia_blacklist.py

127 build_msg DATA "update searx/data/wikidata_units.json"

128 pyenv.cmd python searx_extra/update/update_wikidata_units.py

129 build_msg DATA "update searx/data/currencies.json"

130 pyenv.cmd python searx_extra/update/update_currencies.py

131}

132

133

134data.languages() {

135 ( set -e

136 build_msg ENGINES "fetch languages .."

137 pyenv.cmd python searx_extra/update/update_languages.py

138 build_msg ENGINES "update update searx/languages.py"

139 build_msg DATA "update searx/data/engines_languages.json"

140 )

141 dump_return $?

142}

143

144data.useragents() {

145 build_msg DATA "update searx/data/useragents.json"

146 pyenv.cmd python searx_extra/update/update_firefox_version.py

147 dump_return $?

148}

149

150docs.prebuild() {

151 build_msg DOCS "build ${DOCS_BUILD}/includes"

152 (

153 set -e

154 [ "$VERBOSE" = "1" ] && set -x

155 mkdir -p "${DOCS_BUILD}/includes"

156 ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"

157 ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"

158 ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"

159 )

160 dump_return $?

161}

162

163docker.push() {

164 docker.build push

165}

166

167# shellcheck disable=SC2119

168docker.build() {

169 pyenv.install

170

171 build_msg DOCKER build

172 # run installation in a subprocess and activate pyenv

173

174 # See https://www.shellcheck.net/wiki/SC1001 and others ..

175 # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001

176 ( set -e

177 # shellcheck source=/dev/null

178 source "${PY_ENV_BIN}/activate"

179

180 # Check if it is a git repository

181 if [ ! -d .git ]; then

182 die 1 "This is not Git repository"

183 fi

184 if [ ! -x "$(which git)" ]; then

185 die 1 "git is not installed"

186 fi

187

188 if ! git remote get-url origin 2> /dev/null; then

189 die 1 "there is no remote origin"

190 fi

191

192 # This is a git repository

193

194 # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)

195 # awk to remove the "v" and the "g"

196 SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}')

197

198 # add the suffix "-dirty" if the repository has uncommited change

199 # /!\ HACK for searx/searx: ignore utils/brand.env

200 git update-index -q --refresh

201 if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then

202 SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"

203 fi

204

205 # Get the last git commit id, will be added to the Searx version (see Dockerfile)

206 VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4)

207 build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"

208

209 # Check consistency between the git tag and the searx/version.py file

210 # /! HACK : parse Python file with bash /!

211 # otherwise it is not possible build the docker image without all Python

212 # dependencies ( version.py loads __init__.py )

213 # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")

215 if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then

216 err_msg "git tag: $SEARX_GIT_VERSION"

217 err_msg "searx/version.py: $SEARX_PYTHON_VERSION"

218 die 1 "Inconsistency between the last git tag and the searx/version.py file"

219 fi

220

221 # define the docker image name

222 GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/$[^\/]*$.*/\1/')

223 SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}"

224

225 # build Docker image

226 build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

227 sudo docker build \

228 --build-arg GIT_URL="${GIT_URL}" \

229 --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \

230 --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \

231 --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \

232 --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \

233 --build-arg LABEL_VCS_URL="${GIT_URL}" \

234 --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \

235 --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \

236 -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" .

237

238 if [ "$1" = "push" ]; then

239 sudo docker push "${SEARX_IMAGE_NAME}:latest"

240 sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"

241 fi

242 )

243 dump_return $?

244}

245

246# shellcheck disable=SC2119

247gecko.driver() {

248 pyenv.install

249

250 build_msg INSTALL "gecko.driver"

251 # run installation in a subprocess and activate pyenv

252 ( set -e

253 # shellcheck source=/dev/null

254 source "${PY_ENV_BIN}/activate"

255

256 # TODO : check the current geckodriver version

257 geckodriver -V > /dev/null 2>&1 || NOTFOUND=1

258 set +e

259 if [ -z "$NOTFOUND" ]; then

260 build_msg INSTALL "geckodriver already installed"

261 return

262 fi

263 PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"

264 case "$PLATFORM" in

265 "linux 32bit" | "linux2 32bit") ARCH="linux32";;

266 "linux 64bit" | "linux2 64bit") ARCH="linux64";;

267 "windows 32 bit") ARCH="win32";;

268 "windows 64 bit") ARCH="win64";;

269 "mac 64bit") ARCH="macos";;

270 esac

271 GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";

272

273 build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"

274

275 FILE="$(mktemp)"

276 wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver

277 rm -- "$FILE"

278 chmod 755 -- "${PY_ENV_BIN}/geckodriver"

279 )

280 dump_return $?

281}

282

283node.env() {

284 local err=0

285 pushd "${REPO_ROOT}" &> /dev/null

286 # shellcheck disable=SC2230

287 which npm &> /dev/null || die 1 'node.env - npm is not found!'

288

289 ( set -e

290 # shellcheck disable=SC2030

291 PATH="$(npm bin):$PATH"

292 export PATH

293

294 build_msg INSTALL "npm install $NPM_PACKAGES"

295 # shellcheck disable=SC2086

296 npm install $NPM_PACKAGES

297

298 cd "${REPO_ROOT}/searx/static/themes/oscar"

299 build_msg INSTALL "($(pwd)) npm install"

300 npm install

301

302 build_msg INSTALL "($(pwd)) npm install"

303 cd "${REPO_ROOT}/searx/static/themes/simple"

304 npm install

305 )

306 err=$?

307 popd &> /dev/null

308 dump_return "$err"

309}

310

311node.clean() {

312

313 build_msg CLEAN "locally installed npm dependencies"

314 rm -rf \

315 ./node_modules \

316 ./package-lock.json \

317 ./searx/static/themes/oscar/package-lock.json \

318 ./searx/static/themes/oscar/node_modules \

319 ./searx/static/themes/simple/package-lock.json \

320 ./searx/static/themes/simple/node_modules

321 dump_return $?

322}

323

324py.build() {

325 build_msg BUILD "[pylint] python package ${PYDIST}"

326 pyenv.cmd python setup.py \

327 sdist -d "${PYDIST}" \

328 bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"

329}

330

331py.clean() {

332 build_msg CLEAN pyenv

333 ( set -e

334 pyenv.drop

335 [ "$VERBOSE" = "1" ] && set -x

336 rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info

337 find . -name '*.pyc' -exec rm -f {} +

338 find . -name '*.pyo' -exec rm -f {} +

339 find . -name __pycache__ -exec rm -rf {} +

340 )

341}

342

343pyenv.check() {

344 cat <<EOF

345import yaml

346print('import yaml --> OK')

347EOF

348}

349

350pyenv.install() {

351

352 if ! pyenv.OK; then

353 py.clean > /dev/null

354 fi

355 if pyenv.install.OK > /dev/null; then

356 return 0

357 fi

358 ( set -e

359 pyenv

360 build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"

361 "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"

362 buildenv

363 ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})"

364}

365

366pyenv.uninstall() {

367 build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"

368 pyenv.cmd python setup.py develop --uninstall 2>&1 \

369 | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "

370

371}

372

373pypi.upload() {

374 py.clean

375 py.build

376 # https://github.com/pypa/twine

377 pyenv.cmd twine upload "${PYDIST}"/*

378}

379

380pypi.upload.test() {

381 py.clean

382 py.build

383 pyenv.cmd twine upload -r testpypi "${PYDIST}"/*

384}

385

386test.pylint() {

387 # shellcheck disable=SC2086

388 ( set -e

389 build_msg TEST "[pylint] \$PYLINT_FILES"

390 pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

391 "${PYLINT_FILES[@]}"

392

393 build_msg TEST "[pylint] searx/engines"

394 pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

395 --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

396 --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \

397 searx/engines

398

399 build_msg TEST "[pylint] searx tests"

400 pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \

401 --disable="${PYLINT_SEARX_DISABLE_OPTION}" \

402 --ignore=searx/engines \

403 tests

404 )

405 dump_return $?

406}

407

408test.pep8() {

409 build_msg TEST 'pycodestyle (formerly pep8)'

410 local _exclude=""

411 printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"

412 pyenv.cmd pycodestyle \

413 --exclude="searx/static, searx/languages.py, $_exclude " \

414 --max-line-length=120 \

415 --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \

416 searx tests

417 dump_return $?

418}

419

420test.unit() {

421 build_msg TEST 'tests/unit'

422 pyenv.cmd python -m nose2 -s tests/unit

423 dump_return $?

424}

425

426test.coverage() {

427 build_msg TEST 'unit test coverage'

428 ( set -e

429 pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit

430 pyenv.cmd coverage report

431 pyenv.cmd coverage html

432 )

433 dump_return $?

434}

435

436test.robot() {

437 build_msg TEST 'robot'

438 gecko.driver

439 PYTHONPATH=. pyenv.cmd python searx/testing.py robot

440 dump_return $?

441}

442

443test.clean() {

444 build_msg CLEAN "test stuff"

445 rm -rf geckodriver.log .coverage coverage/

446 dump_return $?

447}

448

449themes.all() {

450 ( set -e

451 node.env

452 themes.oscar

453 themes.simple

454 )

455 dump_return $?

456}

457

458themes.oscar() {

459 local gruntfile=searx/static/themes/oscar/gruntfile.js

460 build_msg GRUNT "${gruntfile}"

461 PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"

462 dump_return $?

463}

464

465themes.simple() {

466 local gruntfile=searx/static/themes/simple/gruntfile.js

467 build_msg GRUNT "${gruntfile}"

468 PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}"

469 dump_return $?

470}

471

472# shellcheck disable=SC2119

473main() {

474

475 local _type

476 local cmd="$1"; shift

477

478 if [ "$cmd" == "" ]; then

479 help

480 err_msg "missing command"

481 return 42

482 fi

483

484 case "$cmd" in

485 --getenv) var="$1"; echo "${!var}";;

486 --help) help;;

487 --*)

488 help

489 err_msg "unknown option $cmd"

490 return 42

491 ;;

492 *)

493 _type="$(type -t "$cmd")"

494 if [ "$_type" != 'function' ]; then

495 err_msg "unknown command $1 / use --help"

496 return 42

497 else

498 "$cmd" "$@"

499 fi

500 ;;

501 esac

502}

503

504main "$@"