on master help sign in
Update Docker image

See #1561 , use uwsgi and Alpine Linux Volume: /var/log/uwsgi contains error log for 2 days (file uwsgi.log) /etc/searx contains the settings.yml and uwsgi.ini files. The docker image creates them if they don't exist. The two files can be modified after the first run. See below. Environement variables: MORTY_URL : external URL of Morty MORTY_KEY : base64 encoded key BASE_URL : external URL of Searx BIND_ADDRESS : internal HTTP port to listen to Labels : org.label-schema.schema.* Parameters: -h : display this help -d : will update the settings and quit immediately (settings.yml and uwsgi.ini) -f : always update the settings (previous version saved with suffix .old). without this parameter, the new settings are copied with suffix .new When the Docker image contains newer settings: - without -f parameter: the new versions are copied to /etc/searx/settings.yml.new and /etc/searx/uwsgi.ini.new. - with -f parameter: the old versions are renamed with .old suffix. The new version replaces /etc/searx/settings.yml and /etc/searx/uwsgi.ini Build using "./manage.sh docker_build", add "push" as parameter also push the Docker image. The script requires a git repository to work (it makes sure that the last git tag matches searx/version.py) "git describe" is used to create a meaningful version. Example : 0.15.0-90-49c5bcb4-dirty (dirty means that the docker image was made with uncommited changes). Use "docker inspect -f {{.Config.Labels.version}} searx" to get the version of an existing image. .dockerignore based on .gitignore .travis.yml: include docker stage

commitfbe40001d35ea5cf74c92f35d76c642a480a5c9f authorDalf <alex@al-f.net> dateSat 29 Jun 2019, 11:59 (7 years ago) parent3faaf34
7 files changed+361−46
.dockerignore+41 .gitignore+1 .travis.yml+28−5 Dockerfile+61−41 dockerfiles/docker-entrypoint.sh+128 dockerfiles/uwsgi.ini+33 manage.sh+69
.dockerignorenew file+41
@@ -0,0 +1,41 @@
1+*~
2+*/*~
3+*/*/*~
4+*/*/*/*~
5+*/*/*/*/*~
6+
7+# Git
8+.git
9+.gitignore
10+
11+# CI
12+.codeclimate.yml
13+.travis.yml
14+.taskcluster.yml
15+
16+# Byte-compiled / optimized / DLL files
17+__pycache__/
18+*/__pycache__/
19+*/*/__pycache__/
20+*/*/*/__pycache__/
21+*.py[cod]
22+*/*.py[cod]
23+*/*/*.py[cod]
24+*/*/*/*.py[cod]
25+
26+# to sync with .gitignore
27+.coverage
28+coverage/
29+.installed.cfg
30+engines.cfg
31+env
32+searx-ve
33+robot_log.html
34+robot_output.xml
35+robot_report.html
36+test_basic/
37+setup.cfg
38+
39+node_modules/
40+
41+.tx/
.gitignore+1
@@ -1,3 +1,4 @@
1+# to sync with .dockerignore
12.coverage
23coverage/
34.installed.cfg
.travis.yml+28−5
@@ -1,16 +1,14 @@
1+language: python
12sudo: false
23cache:
34 - pip
45 - npm
56 - directories:
67 - $HOME/.cache/pip
8+
79addons:
810 firefox: "latest"
9−language: python
10−python:
11− - "2.7"
12− - "3.5"
13− - "3.6"
11+
1412before_install:
1513 - "export DISPLAY=:99.0"
1614 - "sh -e /etc/init.d/xvfb start"
@@ -27,6 +25,31 @@ script:
2725after_success:
2826 - ./manage.sh py_test_coverage
2927 - codecov
28+
29+stages:
30+ - test
31+ - name: docker
32+ if: branch = master AND type != pull_request AND env(DOCKER_USERNAME) IS present
33+
34+jobs:
35+ include:
36+ - python: "2.7"
37+ - python: "3.5"
38+ - python: "3.6"
39+ - stage: docker
40+ python: "3.6"
41+ git:
42+ depth: false
43+ services:
44+ - docker
45+ addons: []
46+ before_install: true
47+ install: true
48+ script:
49+ - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
50+ - ./manage.sh docker_build push
51+ after_success: true
52+
3053notifications:
3154 irc:
3255 channels:
Dockerfile+61−41
@@ -1,57 +1,77 @@
1−FROM alpine:3.8
2−LABEL maintainer="searx <https://github.com/asciimoo/searx>"
3−LABEL description="A privacy-respecting, hackable metasearch engine."
1+FROM alpine:3.10
2+
3+ARG VERSION_GITCOMMIT=unknow
4+ARG SEARX_GIT_VERSION=unknow
5+
6+ARG SEARX_GID=1000
7+ARG SEARX_UID=1000
8+
9+ARG TIMESTAMP_SETTINGS=0
10+ARG TIMESTAMP_UWSGI=0
11+ARG LABEL_VCS_REF=
12+ARG LABEL_VCS_URL=
13+
14+ENV BASE_URL= \
15+ MORTY_KEY= \
16+ MORTY_URL=
17+EXPOSE 8080
18+VOLUME /etc/searx
19+VOLUME /var/log/uwsgi
420
5−ENV BASE_URL=False IMAGE_PROXY=False HTTP_PROXY_URL= HTTPS_PROXY_URL=
6−EXPOSE 8888
721WORKDIR /usr/local/searx
8−CMD ["/sbin/tini","--","/usr/local/searx/run.sh"]
9−
10−RUN adduser -D -h /usr/local/searx -s /bin/sh searx searx \
11− && echo '#!/bin/sh' >> run.sh \
12− && echo 'sed -i "s|base_url : False|base_url : $BASE_URL|g" searx/settings.yml' >> run.sh \
13− && echo 'sed -i "s/image_proxy : False/image_proxy : $IMAGE_PROXY/g" searx/settings.yml' >> run.sh \
14− && echo 'sed -i "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml' >> run.sh \
15− && echo 'if [ -n "$HTTP_PROXY_URL" ] || [ -n "$HTTPS_PROXY_URL" ]; then' >> run.sh \
16− && echo ' sed -i "s~^# proxies :~ proxies:\\n http: ${HTTP_PROXY_URL}\\n https: ${HTTPS_PROXY_URL}\\n~" searx/settings.yml' >> run.sh \
17− && echo 'fi' >> run.sh \
18− && echo 'python searx/webapp.py' >> run.sh \
19− && chmod +x run.sh
22+
23+RUN addgroup -g ${SEARX_GID} searx && \
24+ adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx
2025
2126COPY requirements.txt ./requirements.txt
2227
23−RUN echo "@commuedge http://nl.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \
24− && apk -U add \
25− build-base \
26− python \
27− python-dev \
28− py-pip \
29− libxml2 \
30− libxml2-dev \
31− libxslt \
32− libxslt-dev \
33− libffi-dev \
34− openssl \
35− openssl-dev \
36− ca-certificates \
37− tini@commuedge \
38− && pip install --upgrade pip \
39− && pip install --no-cache -r requirements.txt \
40− && apk del \
28+RUN apk -U upgrade \
29+ && apk add -t build-dependencies \
4130 build-base \
42− python-dev \
31+ py3-setuptools \
32+ python3-dev \
4333 libffi-dev \
44− openssl-dev \
4534 libxslt-dev \
4635 libxml2-dev \
4736 openssl-dev \
37+ tar \
38+ git \
39+ && apk add \
4840 ca-certificates \
41+ su-exec \
42+ python3 \
43+ libxml2 \
44+ libxslt \
45+ openssl \
46+ tini \
47+ uwsgi \
48+ uwsgi-python3 \
49+ && pip3 install --upgrade pip \
50+ && pip3 install --no-cache -r requirements.txt \
51+ && apk del build-dependencies \
4952 && rm -f /var/cache/apk/*
5053
51−COPY . .
54+COPY --chown=searx:searx . .
5255
53−RUN chown -R searx:searx *
56+RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \
57+ touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \
58+ touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \
59+ if [ ! -z $VERSION_GITCOMMIT ]; then\
60+ echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \
61+ fi
5462
55−USER searx
63+ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]
5664
57−RUN sed -i "s/127.0.0.1/0.0.0.0/g" searx/settings.yml
65+# Keep this argument at the end since it change each time
66+ARG LABEL_DATE=
67+LABEL maintainer="searx <https://github.com/asciimoo/searx>" \
68+ description="A privacy-respecting, hackable metasearch engine." \
69+ version="${SEARX_GIT_VERSION}" \
70+ org.label-schema.schema-version="1.0" \
71+ org.label-schema.name="searx" \
72+ org.label-schema.schema-version="${SEARX_GIT_VERSION}" \
73+ org.label-schema.url="${LABEL_VCS_URL}" \
74+ org.label-schema.vcs-ref=${LABEL_VCS_REF} \
75+ org.label-schema.vcs-url=${LABEL_VCS_URL} \
76+ org.label-schema.build-date="${LABEL_DATE}" \
77+ org.label-schema.usage="https://github.com/searx/searx-docker"
dockerfiles/docker-entrypoint.shnew file+128
@@ -0,0 +1,128 @@
1+#!/bin/sh
2+
3+export SEARX_VERSION=$(su searx -c 'python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)"')
4+printf 'searx version %s\n\n' "${SEARX_VERSION}"
5+
6+export UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini
7+export SEARX_SETTINGS_PATH=/etc/searx/settings.yml
8+
9+if [ -z "${BIND_ADDRESS}" ]; then
10+ export BIND_ADDRESS=":8080"
11+fi
12+
13+# Parse command line
14+FORCE_CONF_UPDATE=0
15+DRY_RUN=0
16+while getopts "fdh" option
17+do
18+ case $option in
19+ f)
20+ FORCE_CONF_UPDATE=1
21+ ;;
22+ d)
23+ DRY_RUN=1
24+ ;;
25+ h)
26+ printf "Command line:\n\n"
27+ printf " -h Display this help\n"
28+ printf " -d Dry run to update the configuration files.\n"
29+ printf " -f Always update on the configuration files (existing files are renamed with the .old suffix)\n"
30+ printf " Without this option, new configuration files are copied with the .new suffix\n"
31+ printf "\nEnvironment variables:\n\n"
32+ printf " BASE_URL settings.yml : server.base_url\n"
33+ printf " MORTY_URL settings.yml : result_proxy.url\n"
34+ printf " MORTY_KEY settings.yml : result_proxy.key\n"
35+ printf " BIND_ADDRESS where uwsgi will accept HTTP request (format : host:port)\n"
36+ exit 0
37+ esac
38+done
39+
40+# helpers to update the configuration files
41+patch_uwsgi_settings() {
42+ CONF="$1"
43+
44+ # Nothing
45+}
46+
47+patch_searx_settings() {
48+ CONF="$1"
49+
50+ # Make sure that there is trailing slash at the end of BASE_URL
51+ # see http://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion
52+ export BASE_URL="${BASE_URL%/}/"
53+
54+ # update settings.yml
55+ sed -i -e "s|base_url : False|base_url : ${BASE_URL}|g" \
56+ -e "s/ultrasecretkey/$(openssl rand -hex 32)/g" \
57+ "${CONF}"
58+
59+ # Morty configuration
60+ if [ ! -z "${MORTY_KEY}" -a ! -z "${MORTY_URL}" ]; then
61+ sed -i -e "s/image_proxy : False/image_proxy : True/g" \
62+ "${CONF}"
63+ cat >> "${CONF}" <<-EOF
64+
65+# Morty configuration
66+result_proxy:
67+ url : ${MORTY_URL}
68+ key : !!binary "${MORTY_KEY}"
69+EOF
70+ fi
71+}
72+
73+update_conf() {
74+ FORCE_CONF_UPDATE="$1"
75+ CONF="$2"
76+ NEW_CONF="${2}.new"
77+ OLD_CONF="${2}.old"
78+ REF_CONF="$3"
79+ PATCH_REF_CONF="$4"
80+
81+ if [ -f "${CONF}" ]; then
82+ if [ "${REF_CONF}" -nt "${CONF}" ]; then
83+ # There is a new version
84+ if [ $FORCE_CONF_UPDATE ]; then
85+ # Replace the current configuration
86+ printf '⚠️ Automaticaly update %s to the new version\n' "${CONF}"
87+ if [ ! -f "${OLD_CONF}" ]; then
88+ printf 'The previous configuration is saved to %s\n' "${OLD_CONF}"
89+ mv "${CONF}" "${OLD_CONF}"
90+ fi
91+ cp "${REF_CONF}" "${CONF}"
92+ $PATCH_REF_CONF "${CONF}"
93+ else
94+ # Keep the current configuration
95+ printf '⚠️ Check new version %s to make sure searx is working properly\n' "${NEW_CONF}"
96+ cp "${REF_CONF}" "${NEW_CONF}"
97+ $PATCH_REF_CONF "${NEW_CONF}"
98+ fi
99+ else
100+ printf 'Use existing %s\n' "${CONF}"
101+ fi
102+ else
103+ printf 'Create %s\n' "${CONF}"
104+ cp "${REF_CONF}" "${CONF}"
105+ $PATCH_REF_CONF "${CONF}"
106+ fi
107+}
108+
109+# make sure there are uwsgi settings
110+update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searx/dockerfiles/uwsgi.ini" "patch_uwsgi_settings"
111+
112+# make sure there are searx settings
113+update_conf "${FORCE_CONF_UPDATE}" "${SEARX_SETTINGS_PATH}" "/usr/local/searx/searx/settings.yml" "patch_searx_settings"
114+
115+# dry run (to update configuration files, then inspect them)
116+if [ $DRY_RUN -eq 1 ]; then
117+ printf 'Dry run\n'
118+ exit
119+fi
120+
121+#
122+touch /var/run/uwsgi-logrotate
123+chown -R searx:searx /var/log/uwsgi /var/run/uwsgi-logrotate
124+unset MORTY_KEY
125+
126+# Start uwsgi
127+printf 'Listen on %s\n' "${BIND_ADDRESS}"
128+exec su-exec searx:searx uwsgi --master --http-socket "${BIND_ADDRESS}" "${UWSGI_SETTINGS_PATH}"
dockerfiles/uwsgi.ininew file+33
@@ -0,0 +1,33 @@
1+[uwsgi]
2+# Who will run the code
3+uid = searx
4+gid = searx
5+
6+# Number of workers (usually CPU count)
7+workers = 4
8+
9+# The right granted on the created socket
10+chmod-socket = 666
11+
12+# Plugin to use and interpretor config
13+single-interpreter = true
14+master = true
15+plugin = python3
16+lazy-apps = true
17+enable-threads = true
18+
19+# Module to import
20+module = searx.webapp
21+
22+# Virtualenv and python path
23+pythonpath = /usr/local/searx/
24+chdir = /usr/local/searx/searx/
25+
26+# Disable logging for privacy
27+disable-logging=True
28+
29+# But keep errors for 2 days
30+touch-logrotate = /run/uwsgi-logrotate
31+unique-cron = 15 0 -1 -1 -1 { touch /run/uwsgi-logrotate }
32+log-backupname = /var/log/uwsgi/uwsgi.log.1
33+logto = /var/log/uwsgi/uwsgi.log
manage.sh+69
@@ -158,6 +158,74 @@ grunt_build() {
158158 grunt --gruntfile "$SEARX_DIR/static/themes/simple/gruntfile.js"
159159}
160160
161+docker_build() {
162+ # Check if it is a git repository
163+ if [ ! -d .git ]; then
164+ echo "This is not Git repository"
165+ exit 1
166+ fi
167+
168+ if [ ! -x "$(which git)" ]; then
169+ echo "git is not installed"
170+ exit 1
171+ fi
172+
173+ if [ ! git remote get-url origin 2> /dev/null ]; then
174+ echo "there is no remote origin"
175+ exit 1
176+ fi
177+
178+ # This is a git repository
179+
180+ # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e)
181+ # awk to remove the "v" and the "g"
182+ SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); $3=substr($3, 2); print}')
183+
184+ # add the suffix "-dirty" if the repository has uncommited change
185+ git update-index -q --refresh
186+ if [ ! -z "$(git diff-index --name-only HEAD --)" ]; then
187+ SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty"
188+ fi
189+
190+ # Get the last git commit id, will be added to the Searx version (see Dockerfile)
191+ VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4)
192+ echo "Last commit : $VERSION_GITCOMMIT"
193+
194+ # Check consistency between the git tag and the searx/version.py file
195+ # /!\ HACK : parse Python file with bash /!\
196+ # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py )
197+ # SEARX_PYTHON_VERSION=$(python -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")
198+ SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)
199+ if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then
200+ echo "Inconsistency between the last git tag and the searx/version.py file"
201+ echo "git tag: $SEARX_GIT_VERSION"
202+ echo "searx/version.py: $SEARX_PYTHON_VERSION"
203+ exit 1
204+ fi
205+
206+ # define the docker image name
207+ # /!\ HACK to get the user name /!\
208+ GITHUB_USER=$(git remote get-url origin | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
209+ SEARX_IMAGE_NAME="${GITHUB_USER:-searx}/searx"
210+
211+ # build Docker image
212+ echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}"
213+ sudo docker build \
214+ --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \
215+ --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
216+ --build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
217+ --build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \
218+ --build-arg LABEL_VCS_URL=$(git remote get-url origin) \
219+ --build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \
220+ --build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \
221+ -t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} .
222+
223+ if [ "$1" = "push" ]; then
224+ sudo docker push ${SEARX_IMAGE_NAME}:latest
225+ sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}
226+ fi
227+}
228+
161229#
162230# Help
163231#
@@ -182,6 +250,7 @@ Commands
182250 locales - Compile locales
183251 styles - Build less files
184252 grunt_build - Build files for themes
253+ docker_build - Build Docker image
185254
186255 Tests
187256 -----