mirror of
https://github.com/osm-search/Nominatim.git
synced 2024-11-30 02:07:52 +03:00
CI: use vagrant scripts for import tests
Use vanilla docker images of Ubuntu and leave the setup to the vagrant scripts. Then do the usual import tests. Also fixes a couple of issues found with the scripts
This commit is contained in:
Sarah Hoffmann
parent
0fabeefc3e
commit
6ebbbfee61
2
.github/workflows/ci-tests.yml
vendored
2
.github/workflows/ci-tests.yml
vendored
@ -4,6 +4,7 @@ on: [ push, pull_request ]
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
if: False
|
||||
strategy:
|
||||
matrix:
|
||||
ubuntu: [18, 20]
|
||||
@ -120,6 +121,7 @@ jobs:
|
||||
if: matrix.ubuntu == 20
|
||||
|
||||
import:
|
||||
if: False
|
||||
strategy:
|
||||
matrix:
|
||||
ubuntu: [18, 20]
|
||||
|
||||
152
.github/workflows/vagrant-scripts.yml
vendored
Normal file
152
.github/workflows/vagrant-scripts.yml
vendored
Normal file
@ -0,0 +1,152 @@
|
||||
name: Install scripts
|
||||
|
||||
on: [ push, pull_request ]
|
||||
|
||||
jobs:
|
||||
create-archive:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Get Date
|
||||
id: get-date
|
||||
run: |
|
||||
echo "::set-output name=date::$(/bin/date -u "+%Y%W")"
|
||||
shell: bash
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
data/country_grid.sql.gz
|
||||
key: nominatim-country-data-${{ steps.get-date.outputs.date }}
|
||||
|
||||
- name: Package tarball
|
||||
run: |
|
||||
if [ ! -f data/country_grid.sql.gz ]; then
|
||||
wget --no-verbose -O data/country_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
|
||||
fi
|
||||
cd ..
|
||||
tar cjf nominatim-src.tar.bz2 Nominatim
|
||||
mv nominatim-src.tar.bz2 Nominatim
|
||||
|
||||
- name: 'Upload Artifact'
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: full-source
|
||||
path: nominatim-src.tar.bz2
|
||||
retention-days: 1
|
||||
|
||||
|
||||
ubuntu:
|
||||
runs-on: ubuntu-latest
|
||||
needs: create-archive
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
ubuntu: [18, 20]
|
||||
include:
|
||||
- ubuntu: 18
|
||||
install_mode:
|
||||
- ubuntu: 20
|
||||
install_mode: install-apache
|
||||
|
||||
container:
|
||||
image: ubuntu:${{ matrix.ubuntu }}.04
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0}
|
||||
|
||||
steps:
|
||||
- name: Prepare container
|
||||
run: |
|
||||
export APT_LISTCHANGES_FRONTEND=none
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y git sudo wget
|
||||
ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone
|
||||
shell: bash
|
||||
|
||||
- name: Setup import user
|
||||
run: |
|
||||
useradd -m nominatim -G sudo
|
||||
echo '%sudo ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd
|
||||
echo "/home/nominatim/Nominatim/vagrant/Install-on-Ubuntu-$UBUNTU.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh
|
||||
shell: bash
|
||||
env:
|
||||
UBUNTU: ${{ matrix.ubuntu }}
|
||||
INSTALL_MODE: ${{ matrix.install_mode }}
|
||||
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: full-source
|
||||
path: /home/nominatim
|
||||
|
||||
- name: Get Date
|
||||
id: get-date
|
||||
run: |
|
||||
echo "::set-output name=date::$(/bin/date -u "+%Y%W")"
|
||||
shell: bash
|
||||
|
||||
- name: Install Nominatim
|
||||
run: |
|
||||
export USERNAME=nominatim
|
||||
export USERHOME=/home/nominatim
|
||||
export NOSYSTEMD=yes
|
||||
tar xf nominatim-src.tar.bz2
|
||||
. vagrant.sh
|
||||
working-directory: /home/nominatim
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: |
|
||||
monaco-latest.osm.pbf
|
||||
key: nominatim-test-data-${{ steps.get-date.outputs.date }}
|
||||
|
||||
- name: Prepare import environment
|
||||
run: |
|
||||
if [ ! -f monaco-latest.osm.pbf ]; then
|
||||
wget --no-verbose https://download.geofabrik.de/europe/monaco-latest.osm.pbf
|
||||
fi
|
||||
mkdir data-env data-env-reverse
|
||||
chown nominatim. data-env data-env-reverse
|
||||
shell: bash
|
||||
|
||||
- name: Import
|
||||
run: nominatim import --osm-file ../monaco-latest.osm.pbf
|
||||
working-directory: data-env
|
||||
|
||||
- name: Import special phrases
|
||||
run: nominatim special-phrases --import-from-wiki
|
||||
working-directory: data-env
|
||||
|
||||
- name: Check full import
|
||||
run: nominatim admin --check-database
|
||||
working-directory: data-env
|
||||
|
||||
- name: Warm up database
|
||||
run: nominatim admin --warm
|
||||
working-directory: data-env
|
||||
|
||||
- name: Run update
|
||||
run: |
|
||||
sudo apt-get install -y python3-pip
|
||||
pip3 install --user osmium
|
||||
nominatim replication --init
|
||||
NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once
|
||||
working-directory: data-env
|
||||
|
||||
- name: Run reverse-only import
|
||||
run : |
|
||||
echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' > .env
|
||||
nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only --no-updates
|
||||
working-directory: data-env-reverse
|
||||
env:
|
||||
NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse
|
||||
|
||||
- name: Check reverse import
|
||||
run: nominatim admin --check-database
|
||||
working-directory: data-env-reverse
|
||||
@ -1,8 +1,7 @@
|
||||
#!/bin/bash
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# hacks for broken vagrant box #DOCS:
|
||||
sudo rm -f /var/lib/dpkg/lock #DOCS:
|
||||
sudo update-locale LANG=en_US.UTF-8 #DOCS:
|
||||
export APT_LISTCHANGES_FRONTEND=none #DOCS:
|
||||
export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
|
||||
@ -15,10 +14,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
#
|
||||
# These instructions expect that you have a freshly installed Ubuntu 18.04.
|
||||
#
|
||||
# Make sure all packages are are up-to-date by running:
|
||||
# Make sure all packages are up-to-date by running:
|
||||
#
|
||||
|
||||
sudo apt-get -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" --force-yes -fuy install grub-pc #DOCS:
|
||||
sudo apt update -qq
|
||||
|
||||
# Now you can install all packages needed for Nominatim:
|
||||
@ -30,12 +28,12 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
|
||||
postgresql-contrib-10 postgresql-10-postgis-scripts \
|
||||
php php-pgsql php-intl libicu-dev python3-pip \
|
||||
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
|
||||
python3-psutil python3-jinja2 python3-icu git
|
||||
|
||||
# Some of the Python packages that come with Ubuntu 18.04 are too old, so
|
||||
# install the latest version from pip:
|
||||
|
||||
pip3 install python-dotenv datrie pyyaml
|
||||
pip3 install --user python-dotenv datrie pyyaml psycopg2
|
||||
|
||||
#
|
||||
# System Configuration
|
||||
@ -60,8 +58,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
# To be able to copy and paste instructions from this manual, export
|
||||
# user name and home directory now like this:
|
||||
#
|
||||
if [ "x$USERNAME" == "x" ]; then #DOCS:
|
||||
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
|
||||
export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim
|
||||
fi #DOCS:
|
||||
#
|
||||
# **Never, ever run the installation as a root user.** You have been warned.
|
||||
#
|
||||
@ -79,7 +79,11 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
#
|
||||
# Restart the postgresql service after updating this config file.
|
||||
|
||||
sudo systemctl restart postgresql
|
||||
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
|
||||
sudo pg_ctlcluster 10 main start
|
||||
else #DOCS:
|
||||
sudo systemctl restart postgresql #DOCS:
|
||||
fi #DOCS:
|
||||
|
||||
#
|
||||
# Finally, we need to add two postgres users: one for the user that does
|
||||
@ -133,11 +137,13 @@ fi #DOCS:
|
||||
#
|
||||
# The webserver should serve the php scripts from the website directory of your
|
||||
# [project directory](../admin/Import.md#creating-the-project-directory).
|
||||
# Therefore set up a project directory and populate the website directory:
|
||||
# This directory needs to exist when being configured.
|
||||
# Therefore set up a project directory and create the website directory:
|
||||
|
||||
mkdir $USERHOME/nominatim-project
|
||||
cd $USERHOME/nominatim-project
|
||||
nominatim refresh --website
|
||||
mkdir $USERHOME/nominatim-project/website
|
||||
|
||||
# The import process will populate the directory later.
|
||||
#
|
||||
# Option 1: Using Apache
|
||||
# ----------------------
|
||||
@ -170,7 +176,11 @@ EOFAPACHECONF
|
||||
#
|
||||
|
||||
sudo a2enconf nominatim
|
||||
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
|
||||
sudo apache2ctl start #DOCS:
|
||||
else #DOCS:
|
||||
sudo systemctl restart apache2
|
||||
fi #DOCS:
|
||||
|
||||
# The Nominatim API is now available at `http://localhost/nominatim/`.
|
||||
|
||||
@ -250,7 +260,12 @@ EOF_NGINX_CONF
|
||||
# Enable the configuration and restart Nginx
|
||||
#
|
||||
|
||||
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
|
||||
sudo /usr/sbin/php-fpm7.2 --nodaemonize --fpm-config /etc/php/7.2/fpm/php-fpm.conf & #DOCS:
|
||||
sudo /usr/sbin/nginx & #DOCS:
|
||||
else #DOCS:
|
||||
sudo systemctl restart php7.2-fpm nginx
|
||||
fi #DOCS:
|
||||
|
||||
# The Nominatim API is now available at `http://localhost/`.
|
||||
|
||||
|
||||
@ -1,8 +1,7 @@
|
||||
#!/bin/bash
|
||||
#!/bin/bash -e
|
||||
#
|
||||
# hacks for broken vagrant box #DOCS:
|
||||
sudo rm -f /var/lib/dpkg/lock #DOCS:
|
||||
sudo update-locale LANG=en_US.UTF-8 #DOCS:
|
||||
export APT_LISTCHANGES_FRONTEND=none #DOCS:
|
||||
export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
|
||||
@ -14,14 +13,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
#
|
||||
# These instructions expect that you have a freshly installed Ubuntu 20.04.
|
||||
#
|
||||
# Make sure all packages are are up-to-date by running:
|
||||
# Make sure all packages are up-to-date by running:
|
||||
#
|
||||
|
||||
#DOCS: :::sh
|
||||
sudo apt-get \ #DOCS:
|
||||
-o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" \ #DOCS:
|
||||
--allow-downgrades --allow-remove-essential --allow-change-held-packages \ #DOCS:
|
||||
-fuy install grub-pc #DOCS:
|
||||
sudo apt update -qq
|
||||
|
||||
# Now you can install all packages needed for Nominatim:
|
||||
@ -59,8 +53,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
# To be able to copy and paste instructions from this manual, export
|
||||
# user name and home directory now like this:
|
||||
#
|
||||
if [ "x$USERNAME" == "x" ]; then #DOCS:
|
||||
export USERNAME=vagrant #DOCS: export USERNAME=nominatim
|
||||
export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim
|
||||
fi #DOCS:
|
||||
#
|
||||
# **Never, ever run the installation as a root user.** You have been warned.
|
||||
#
|
||||
@ -78,8 +74,11 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
|
||||
#
|
||||
# Restart the postgresql service after updating this config file.
|
||||
|
||||
sudo systemctl restart postgresql
|
||||
|
||||
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
|
||||
sudo pg_ctlcluster 12 main start
|
||||
else #DOCS:
|
||||
sudo systemctl restart postgresql #DOCS:
|
||||
fi #DOCS:
|
||||
#
|
||||
# Finally, we need to add two postgres users: one for the user that does
|
||||
# the import and another for the webserver which should access the database
|
||||
@ -131,11 +130,13 @@ fi #DOCS:
|
||||
#
|
||||
# The webserver should serve the php scripts from the website directory of your
|
||||
# [project directory](../admin/Import.md#creating-the-project-directory).
|
||||
# Therefore set up a project directory and populate the website directory:
|
||||
# This directory needs to exist when being configured.
|
||||
# Therefore set up a project directory and create a website directory:
|
||||
|
||||
mkdir $USERHOME/nominatim-project
|
||||
cd $USERHOME/nominatim-project
|
||||
nominatim refresh --website
|
||||
mkdir $USERHOME/nominatim-project/website
|
||||
|
||||
# The import process will populate the directory later.
|
||||
|
||||
#
|
||||
# Option 1: Using Apache
|
||||
@ -169,7 +170,11 @@ EOFAPACHECONF
|
||||
#
|
||||
|
||||
sudo a2enconf nominatim
|
||||
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
|
||||
sudo apache2ctl start #DOCS:
|
||||
else #DOCS:
|
||||
sudo systemctl restart apache2
|
||||
fi #DOCS:
|
||||
|
||||
# The Nominatim API is now available at `http://localhost/nominatim/`.
|
||||
|
||||
@ -252,7 +257,12 @@ EOF_NGINX_CONF
|
||||
# Enable the configuration and restart Nginx
|
||||
#
|
||||
|
||||
if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS:
|
||||
sudo /usr/sbin/php-fpm7.3 --nodaemonize --fpm-config /etc/php/7.3/fpm/php-fpm.conf & #DOCS:
|
||||
sudo /usr/sbin/nginx & #DOCS:
|
||||
else #DOCS:
|
||||
sudo systemctl restart php7.4-fpm nginx
|
||||
fi #DOCS:
|
||||
|
||||
# The Nominatim API is now available at `http://localhost/`.
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user