Merge pull request #2434 from lonvia/vagrant-scripts-in-actions

Test installation instructions via CI
This commit is contained in:
Sarah Hoffmann 2021-08-29 10:11:59 +02:00 committed by GitHub
commit 4c6d674e03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 221 additions and 123 deletions

View File

@ -3,7 +3,38 @@ name: CI Tests
on: [ push, pull_request ]
jobs:
create-archive:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: actions/cache@v2
with:
path: |
data/country_osm_grid.sql.gz
key: nominatim-country-data-1
- name: Package tarball
run: |
if [ ! -f data/country_osm_grid.sql.gz ]; then
wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
fi
cd ..
tar czf nominatim-src.tar.bz2 Nominatim
mv nominatim-src.tar.bz2 Nominatim
- name: 'Upload Artifact'
uses: actions/upload-artifact@v2
with:
name: full-source
path: nominatim-src.tar.bz2
retention-days: 1
tests:
needs: create-archive
strategy:
matrix:
ubuntu: [18, 20]
@ -22,10 +53,12 @@ jobs:
runs-on: ubuntu-${{ matrix.ubuntu }}.04
steps:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
submodules: true
path: Nominatim
name: full-source
- name: Unpack Nominatim
run: tar xf nominatim-src.tar.bz2
- name: Setup PHP
uses: shivammathur/setup-php@v2
@ -39,18 +72,6 @@ jobs:
python-version: 3.6
if: matrix.ubuntu == 18
- name: Get Date
id: get-date
run: |
echo "::set-output name=date::$(/bin/date -u "+%Y%W")"
shell: bash
- uses: actions/cache@v2
with:
path: |
country_grid.sql.gz
key: nominatim-country-data-${{ steps.get-date.outputs.date }}
- uses: ./Nominatim/.github/actions/setup-postgresql
with:
postgresql-version: ${{ matrix.postgresql }}
@ -65,8 +86,7 @@ jobs:
if: matrix.ubuntu == 20
- name: Install test prerequsites
run: |
pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6
run: pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6
if: matrix.ubuntu == 18
- name: PHP linting
@ -119,99 +139,130 @@ jobs:
verbose: true
if: matrix.ubuntu == 20
import:
install:
runs-on: ubuntu-latest
needs: create-archive
strategy:
matrix:
ubuntu: [18, 20]
name: [Ubuntu-18, Ubuntu-20, Centos-8]
include:
- ubuntu: 18
postgresql: 9.5
postgis: 2.5
- ubuntu: 20
postgresql: 13
postgis: 3
- name: Ubuntu-18
flavour: ubuntu
image: "ubuntu:18.04"
ubuntu: 18
install_mode: install-nginx
- name: Ubuntu-20
flavour: ubuntu
image: "ubuntu:20.04"
ubuntu: 20
install_mode: install-apache
- name: Centos-8
flavour: centos
image: "centos:8"
runs-on: ubuntu-${{ matrix.ubuntu }}.04
container:
image: ${{ matrix.image }}
env:
LANG: en_US.UTF-8
defaults:
run:
shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0}
steps:
- uses: actions/checkout@v2
with:
submodules: true
path: Nominatim
- name: Get Date
id: get-date
- name: Prepare container (Ubuntu)
run: |
echo "::set-output name=date::$(/bin/date -u "+%Y%W")"
export APT_LISTCHANGES_FRONTEND=none
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y git sudo wget
ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
shell: bash
if: matrix.flavour == 'ubuntu'
- uses: actions/cache@v2
with:
path: |
country_grid.sql.gz
key: nominatim-country-data-${{ steps.get-date.outputs.date }}
- uses: actions/cache@v2
with:
path: |
monaco-latest.osm.pbf
key: nominatim-test-data-${{ steps.get-date.outputs.date }}
- uses: actions/setup-python@v2
with:
python-version: 3.6
if: matrix.ubuntu == 18
- uses: ./Nominatim/.github/actions/setup-postgresql
with:
postgresql-version: ${{ matrix.postgresql }}
postgis-version: ${{ matrix.postgis }}
- uses: ./Nominatim/.github/actions/build-nominatim
with:
ubuntu: ${{ matrix.ubuntu }}
- name: Clean installation
run: rm -rf Nominatim build
- name: Prepare container (CentOS)
run: |
dnf update -y
dnf install -y sudo
shell: bash
if: matrix.flavour == 'centos'
- name: Setup import user
run: |
useradd -m nominatim
echo 'nominatim ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nominiatim
echo "/home/nominatim/Nominatim/vagrant/Install-on-${OS}.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh
shell: bash
env:
OS: ${{ matrix.name }}
INSTALL_MODE: ${{ matrix.install_mode }}
- uses: actions/download-artifact@v2
with:
name: full-source
path: /home/nominatim
- name: Install Nominatim
run: |
export USERNAME=nominatim
export USERHOME=/home/nominatim
export NOSYSTEMD=yes
export HAVE_SELINUX=no
tar xf nominatim-src.tar.bz2
. vagrant.sh
working-directory: /home/nominatim
- name: Prepare import environment
run: |
if [ ! -f monaco-latest.osm.pbf ]; then
wget --no-verbose https://download.geofabrik.de/europe/monaco-latest.osm.pbf
fi
mkdir data-env
cd data-env
shell: bash
mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf
rm -rf Nominatim
mkdir data-env-reverse
working-directory: /home/nominatim
- name: Prepare import environment (CentOS)
run: |
sudo ln -s /usr/local/bin/nominatim /usr/bin/nominatim
echo NOMINATIM_DATABASE_WEBUSER="apache" > nominatim-project/.env
cp nominatim-project/.env data-env-reverse/.env
working-directory: /home/nominatim
if: matrix.flavour == 'centos'
- name: Import
run: nominatim import --osm-file ../monaco-latest.osm.pbf
shell: bash
working-directory: data-env
run: nominatim import --osm-file ../test.pbf
working-directory: /home/nominatim/nominatim-project
- name: Import special phrases
run: nominatim special-phrases --import-from-wiki
working-directory: data-env
working-directory: /home/nominatim/nominatim-project
- name: Check full import
run: nominatim admin --check-database
working-directory: data-env
working-directory: /home/nominatim/nominatim-project
- name: Warm up database
run: nominatim admin --warm
working-directory: data-env
working-directory: /home/nominatim/nominatim-project
- name: Prepare update (Ubuntu)
run: apt-get install -y python3-pip
shell: bash
if: matrix.flavour == 'ubuntu'
- name: Run update
run: |
nominatim replication --init
nominatim replication --once
working-directory: data-env
pip3 install --user osmium
nominatim replication --init
NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once
working-directory: /home/nominatim/nominatim-project
- name: Run reverse-only import
run : nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only --no-updates
working-directory: data-env
env:
NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse
run : |
echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' >> .env
nominatim import --osm-file ../test.pbf --reverse-only --no-updates
working-directory: /home/nominatim/data-env-reverse
- name: Check reverse import
run: nominatim admin --check-database
working-directory: data-env
working-directory: /home/nominatim/data-env-reverse

View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/bash -ex
#
# *Note:* these installation instructions are also available in executable
# form for use with vagrant under `vagrant/Install-on-Centos-8.sh`.
@ -18,10 +18,12 @@
sudo dnf install -y epel-release redhat-rpm-config
# EPEL contains Postgres 9.6 and 10, but not PostGIS. Postgres 9.4+/10/11/12
# and PostGIS 2.4/2.5/3.0 are availble from postgresql.org
# and PostGIS 2.4/2.5/3.0 are availble from postgresql.org. Enable these
# repositories and make sure, the binaries can be found:
sudo dnf -qy module disable postgresql
sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
export PATH=/usr/pgsql-12/bin:$PATH
# Now you can install all packages needed for Nominatim:
@ -33,9 +35,10 @@
php-pgsql php php-intl php-json libpq-devel \
bzip2-devel proj-devel boost-devel \
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel libicu-dev
python3-psycopg2 \
expat-devel zlib-devel libicu-devel
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml
pip3 install --user python-dotenv psutil Jinja2 PyICU datrie pyyaml
#
@ -54,16 +57,19 @@
# we assume this user is called nominatim and the installation will be in
# /srv/nominatim. To create the user and directory run:
#
sudo mkdir -p /srv/nominatim #DOCS: sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim
sudo chown vagrant /srv/nominatim #DOCS:
# sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim
#
# You may find a more suitable location if you wish.
#
# To be able to copy and paste instructions from this manual, export
# user name and home directory now like this:
#
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
if [ "x$USERNAME" == "x" ]; then #DOCS:
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
export USERHOME=/srv/nominatim
sudo mkdir -p /srv/nominatim #DOCS:
sudo chown vagrant /srv/nominatim #DOCS:
fi #DOCS:
#
# **Never, ever run the installation as a root user.** You have been warned.
#
@ -75,21 +81,29 @@ sudo chown vagrant /srv/nominatim #DOCS:
# ---------------------
#
# CentOS does not automatically create a database cluster. Therefore, start
# with initializing the database, then enable the server to start at boot:
# with initializing the database:
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo -u postgres /usr/pgsql-12/bin/pg_ctl initdb -D /var/lib/pgsql/12/data #DOCS:
sudo mkdir /var/log/postgresql #DOCS:
sudo chown postgres. /var/log/postgresql #DOCS:
else #DOCS:
sudo /usr/pgsql-12/bin/postgresql-12-setup initdb
sudo systemctl enable postgresql-12
fi #DOCS:
#
# Next tune the postgresql configuration, which is located in
# `/var/lib/pgsql/data/postgresql.conf`. See section *Postgres Tuning* in
# Next tune the postgresql configuration, which is located in
# `/var/lib/pgsql/12/data/postgresql.conf`. See section *Postgres Tuning* in
# [the installation page](../admin/Installation.md#postgresql-tuning)
# for the parameters to change.
#
# Now start the postgresql service after updating this config file.
# Now start the postgresql service after updating this config file:
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo -u postgres /usr/pgsql-12/bin/pg_ctl -D /var/lib/pgsql/12/data -l /var/log/postgresql/postgresql-12.log start #DOCS:
else #DOCS:
sudo systemctl enable postgresql-12
sudo systemctl restart postgresql-12
fi #DOCS:
#
# Finally, we need to add two postgres users: one for the user that does
@ -140,11 +154,11 @@ fi #DOCS:
#
# The webserver should serve the php scripts from the website directory of your
# [project directory](../admin/Import.md#creating-the-project-directory).
# Therefore set up a project directory and populate the website directory:
# This directory needs to exist when the webserver is configured.
# Therefore set up a project directory and create the website directory:
#
mkdir $USERHOME/nominatim-project
cd $USERHOME/nominatim-project
nominatim refresh --website
mkdir $USERHOME/nominatim-project/website
#
# You need to create an alias to the website directory in your apache
# configuration. Add a separate nominatim configuration to your webserver:
@ -165,11 +179,15 @@ EOFAPACHECONF
sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS:
#
# Then reload apache
# Then reload apache:
#
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo httpd #DOCS:
else #DOCS:
sudo systemctl enable httpd
sudo systemctl restart httpd
fi #DOCS:
#
# Adding SELinux Security Settings
@ -179,18 +197,19 @@ sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS:
# with a web server accessible from the Internet. At a minimum the
# following SELinux labeling should be done for Nominatim:
if [ "x$HAVE_SELINUX" != "xno" ]; then #DOCS:
sudo semanage fcontext -a -t httpd_sys_content_t "/usr/local/nominatim/lib/lib-php(/.*)?"
sudo semanage fcontext -a -t httpd_sys_content_t "$USERHOME/nominatim-project/website(/.*)?"
sudo semanage fcontext -a -t lib_t "$USERHOME/nominatim-project/module/nominatim.so"
sudo restorecon -R -v /usr/local/lib/nominatim
sudo restorecon -R -v $USERHOME/nominatim-project
fi #DOCS:
# You need to create a minimal configuration file that tells nominatim
# the name of your webserver user:
#DOCS:```sh
echo NOMINATIM_DATABASE_WEBUSER="apache" | tee .env
echo NOMINATIM_DATABASE_WEBUSER="apache" | tee $USERHOME/nominatim-project/.env
#DOCS:```

View File

@ -1,8 +1,7 @@
#!/bin/bash
#!/bin/bash -e
#
# hacks for broken vagrant box #DOCS:
sudo rm -f /var/lib/dpkg/lock #DOCS:
sudo update-locale LANG=en_US.UTF-8 #DOCS:
export APT_LISTCHANGES_FRONTEND=none #DOCS:
export DEBIAN_FRONTEND=noninteractive #DOCS:
@ -15,10 +14,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# These instructions expect that you have a freshly installed Ubuntu 18.04.
#
# Make sure all packages are are up-to-date by running:
# Make sure all packages are up-to-date by running:
#
sudo apt-get -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" --force-yes -fuy install grub-pc #DOCS:
sudo apt update -qq
# Now you can install all packages needed for Nominatim:
@ -30,12 +28,12 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php php-pgsql php-intl libicu-dev python3-pip \
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
python3-psutil python3-jinja2 python3-icu git
# Some of the Python packages that come with Ubuntu 18.04 are too old, so
# install the latest version from pip:
pip3 install python-dotenv datrie pyyaml
pip3 install --user python-dotenv datrie pyyaml psycopg2-binary
#
# System Configuration
@ -60,8 +58,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
# To be able to copy and paste instructions from this manual, export
# user name and home directory now like this:
#
if [ "x$USERNAME" == "x" ]; then #DOCS:
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim
fi #DOCS:
#
# **Never, ever run the installation as a root user.** You have been warned.
#
@ -79,7 +79,11 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# Restart the postgresql service after updating this config file.
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo pg_ctlcluster 10 main start #DOCS:
else #DOCS:
sudo systemctl restart postgresql
fi #DOCS:
#
# Finally, we need to add two postgres users: one for the user that does
@ -133,11 +137,13 @@ fi #DOCS:
#
# The webserver should serve the php scripts from the website directory of your
# [project directory](../admin/Import.md#creating-the-project-directory).
# Therefore set up a project directory and populate the website directory:
# This directory needs to exist when being configured.
# Therefore set up a project directory and create the website directory:
mkdir $USERHOME/nominatim-project
cd $USERHOME/nominatim-project
nominatim refresh --website
mkdir $USERHOME/nominatim-project/website
# The import process will populate the directory later.
#
# Option 1: Using Apache
# ----------------------
@ -166,11 +172,18 @@ EOFAPACHECONF
#DOCS:```
#
# Then enable the configuration and restart apache
# Then enable the configuration with
#
sudo a2enconf nominatim
# and restart apache:
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo apache2ctl start #DOCS:
else #DOCS:
sudo systemctl restart apache2
fi #DOCS:
# The Nominatim API is now available at `http://localhost/nominatim/`.
@ -250,7 +263,12 @@ EOF_NGINX_CONF
# Enable the configuration and restart Nginx
#
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo /usr/sbin/php-fpm7.2 --nodaemonize --fpm-config /etc/php/7.2/fpm/php-fpm.conf & #DOCS:
sudo /usr/sbin/nginx & #DOCS:
else #DOCS:
sudo systemctl restart php7.2-fpm nginx
fi #DOCS:
# The Nominatim API is now available at `http://localhost/`.

View File

@ -1,8 +1,7 @@
#!/bin/bash
#!/bin/bash -e
#
# hacks for broken vagrant box #DOCS:
sudo rm -f /var/lib/dpkg/lock #DOCS:
sudo update-locale LANG=en_US.UTF-8 #DOCS:
export APT_LISTCHANGES_FRONTEND=none #DOCS:
export DEBIAN_FRONTEND=noninteractive #DOCS:
@ -14,14 +13,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# These instructions expect that you have a freshly installed Ubuntu 20.04.
#
# Make sure all packages are are up-to-date by running:
# Make sure all packages are up-to-date by running:
#
#DOCS: :::sh
sudo apt-get \ #DOCS:
-o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" \ #DOCS:
--allow-downgrades --allow-remove-essential --allow-change-held-packages \ #DOCS:
-fuy install grub-pc #DOCS:
sudo apt update -qq
# Now you can install all packages needed for Nominatim:
@ -59,8 +53,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
# To be able to copy and paste instructions from this manual, export
# user name and home directory now like this:
#
if [ "x$USERNAME" == "x" ]; then #DOCS:
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim
fi #DOCS:
#
# **Never, ever run the installation as a root user.** You have been warned.
#
@ -78,8 +74,11 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
#
# Restart the postgresql service after updating this config file.
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo pg_ctlcluster 12 main start #DOCS:
else #DOCS:
sudo systemctl restart postgresql
fi #DOCS:
#
# Finally, we need to add two postgres users: one for the user that does
# the import and another for the webserver which should access the database
@ -131,11 +130,13 @@ fi #DOCS:
#
# The webserver should serve the php scripts from the website directory of your
# [project directory](../admin/Import.md#creating-the-project-directory).
# Therefore set up a project directory and populate the website directory:
# This directory needs to exist when being configured.
# Therefore set up a project directory and create a website directory:
mkdir $USERHOME/nominatim-project
cd $USERHOME/nominatim-project
nominatim refresh --website
mkdir $USERHOME/nominatim-project/website
# The import process will populate the directory later.
#
# Option 1: Using Apache
@ -169,7 +170,11 @@ EOFAPACHECONF
#
sudo a2enconf nominatim
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo apache2ctl start #DOCS:
else #DOCS:
sudo systemctl restart apache2
fi #DOCS:
# The Nominatim API is now available at `http://localhost/nominatim/`.
@ -252,7 +257,12 @@ EOF_NGINX_CONF
# Enable the configuration and restart Nginx
#
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
sudo /usr/sbin/php-fpm7.3 --nodaemonize --fpm-config /etc/php/7.3/fpm/php-fpm.conf & #DOCS:
sudo /usr/sbin/nginx & #DOCS:
else #DOCS:
sudo systemctl restart php7.4-fpm nginx
fi #DOCS:
# The Nominatim API is now available at `http://localhost/`.