From a6dab5e300de9c5664f714eb2c9290d18f01067f Mon Sep 17 00:00:00 2001 From: Marc Tobias Date: Mon, 2 May 2022 18:16:08 +0200 Subject: [PATCH 1/2] Vagrant and CI tests for Ubuntu 22.04 --- .github/workflows/ci-tests.yml | 24 ++- docs/CMakeLists.txt | 1 + docs/mkdocs.yml | 1 + vagrant/Install-on-Ubuntu-22.sh | 271 ++++++++++++++++++++++++++++++++ 4 files changed, 292 insertions(+), 5 deletions(-) create mode 100755 vagrant/Install-on-Ubuntu-22.sh diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4ce14f93..e3a01aa0 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -37,7 +37,7 @@ jobs: needs: create-archive strategy: matrix: - ubuntu: [18, 20] + ubuntu: [18, 20, 22] include: - ubuntu: 18 postgresql: 9.6 @@ -49,6 +49,11 @@ jobs: postgis: 3 pytest: py.test-3 php: 7.4 + - ubuntu: 22 + postgresql: 14 + postgis: 3 + pytest: py.test-3 + php: 8.1 runs-on: ubuntu-${{ matrix.ubuntu }}.04 @@ -85,8 +90,12 @@ jobs: if: matrix.ubuntu == 20 - name: Install test prerequsites - run: pip3 install pytest behave==1.2.6 - if: matrix.ubuntu == 18 + run: pip3 install pylint pytest behave==1.2.6 + if: ${{ (matrix.ubuntu == 18) || (matrix.ubuntu == 22) }} + + - name: Install test prerequsites + run: sudo apt-get install -y -qq python3-pytest + if: matrix.ubuntu == 22 - name: Install latest pylint run: pip3 install pylint @@ -102,7 +111,7 @@ jobs: - name: PHP unit tests run: phpunit ./ working-directory: Nominatim/test/php - if: matrix.ubuntu == 20 + if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }} - name: Python unit tests run: $PYTEST test/python @@ -161,7 +170,7 @@ jobs: strategy: matrix: - name: [Ubuntu-18, Ubuntu-20] + name: [Ubuntu-18, Ubuntu-20, Ubuntu-22] include: - name: Ubuntu-18 flavour: ubuntu @@ -173,6 +182,11 @@ jobs: image: "ubuntu:20.04" ubuntu: 20 install_mode: install-apache + - name: Ubuntu-22 + flavour: ubuntu + image: "ubuntu:22.04" + ubuntu: 22 + install_mode: install-apache container: image: ${{ matrix.image }} diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 3b9a316b..0ccc5974 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -26,6 +26,7 @@ ADD_CUSTOM_TARGET(doc COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Centos-8.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Centos-8.md COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-18.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-18.md COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-20.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-20.md + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-22.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-22.md COMMAND PYTHONPATH=${PROJECT_SOURCE_DIR} mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml ) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index a156fb71..48fe1d0d 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -46,6 +46,7 @@ nav: - 'Installation on CentOS 8' : 'appendix/Install-on-Centos-8.md' - 'Installation on Ubuntu 18' : 'appendix/Install-on-Ubuntu-18.md' - 'Installation on Ubuntu 20' : 'appendix/Install-on-Ubuntu-20.md' + - 'Installation on Ubuntu 22' : 'appendix/Install-on-Ubuntu-22.md' markdown_extensions: - codehilite - admonition diff --git a/vagrant/Install-on-Ubuntu-22.sh b/vagrant/Install-on-Ubuntu-22.sh new file mode 100755 index 00000000..dbb70ffe --- /dev/null +++ b/vagrant/Install-on-Ubuntu-22.sh @@ -0,0 +1,271 @@ +#!/bin/bash -e +# +# hacks for broken vagrant box #DOCS: +sudo rm -f /var/lib/dpkg/lock #DOCS: +export APT_LISTCHANGES_FRONTEND=none #DOCS: +export DEBIAN_FRONTEND=noninteractive #DOCS: + +# *Note:* these installation instructions are also available in executable +# form for use with vagrant under vagrant/Install-on-Ubuntu-22.sh. +# +# Installing the Required Software +# ================================ +# +# These instructions expect that you have a freshly installed Ubuntu 22.04. +# +# Make sure all packages are up-to-date by running: +# + + sudo apt update -qq + +# Now you can install all packages needed for Nominatim: + + sudo apt install -y php-cgi + sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \ + libboost-filesystem-dev libexpat1-dev zlib1g-dev \ + libbz2-dev libpq-dev libproj-dev \ + postgresql-server-dev-14 postgresql-14-postgis-3 \ + postgresql-contrib-14 postgresql-14-postgis-3-scripts \ + php php-pgsql php-intl libicu-dev python3-dotenv \ + python3-psycopg2 python3-psutil python3-jinja2 \ + python3-icu python3-datrie git + +# +# System Configuration +# ==================== +# +# The following steps are meant to configure a fresh Ubuntu installation +# for use with Nominatim. You may skip some of the steps if you have your +# OS already configured. +# +# Creating Dedicated User Accounts +# -------------------------------- +# +# Nominatim will run as a global service on your machine. It is therefore +# best to install it under its own separate user account. In the following +# we assume this user is called nominatim and the installation will be in +# /srv/nominatim. To create the user and directory run: +# +# sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim +# +# You may find a more suitable location if you wish. +# +# To be able to copy and paste instructions from this manual, export +# user name and home directory now like this: +# +if [ "x$USERNAME" == "x" ]; then #DOCS: + export USERNAME=vagrant #DOCS: export USERNAME=nominatim + export USERHOME=/home/vagrant #DOCS: export USERHOME=/srv/nominatim +fi #DOCS: +# +# **Never, ever run the installation as a root user.** You have been warned. +# +# Make sure that system servers can read from the home directory: + + chmod a+x $USERHOME + +# Setting up PostgreSQL +# --------------------- +# +# Tune the postgresql configuration, which is located in +# `/etc/postgresql/14/main/postgresql.conf`. See section *Postgres Tuning* in +# [the installation page](../admin/Installation.md#postgresql-tuning) +# for the parameters to change. +# +# Restart the postgresql service after updating this config file. + +if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: + sudo pg_ctlcluster 14 main start #DOCS: +else #DOCS: + sudo systemctl restart postgresql +fi #DOCS: +# +# Finally, we need to add two postgres users: one for the user that does +# the import and another for the webserver which should access the database +# for reading only: +# + + sudo -u postgres createuser -s $USERNAME + sudo -u postgres createuser www-data + +# +# Installing Nominatim +# ==================== +# +# Building and Configuration +# -------------------------- +# +# Get the source code from Github and change into the source directory +# +if [ "x$1" == "xyes" ]; then #DOCS: :::sh + cd $USERHOME + git clone --recursive https://github.com/openstreetmap/Nominatim.git + cd Nominatim +else #DOCS: + cd $USERHOME/Nominatim #DOCS: +fi #DOCS: + +# When installing the latest source from github, you also need to +# download the country grid: + +if [ ! -f data/country_osm_grid.sql.gz ]; then #DOCS: :::sh + wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz +fi #DOCS: + +# The code must be built in a separate directory. Create this directory, +# then configure and build Nominatim in there: + + mkdir $USERHOME/build + cd $USERHOME/build + cmake $USERHOME/Nominatim + make + sudo make install + +# Nominatim is now ready to use. You can continue with +# [importing a database from OSM data](../admin/Import.md). If you want to set up +# a webserver first, continue reading. +# +# Setting up a webserver +# ====================== +# +# The webserver should serve the php scripts from the website directory of your +# [project directory](../admin/Import.md#creating-the-project-directory). +# This directory needs to exist when being configured. +# Therefore set up a project directory and create a website directory: + + mkdir $USERHOME/nominatim-project + mkdir $USERHOME/nominatim-project/website + +# The import process will populate the directory later. + +# +# Option 1: Using Apache +# ---------------------- +# +if [ "x$2" == "xinstall-apache" ]; then #DOCS: +# +# Apache has a PHP module that can be used to serve Nominatim. To install them +# run: + + sudo apt install -y apache2 libapache2-mod-php + +# You need to create an alias to the website directory in your apache +# configuration. Add a separate nominatim configuration to your webserver: + +#DOCS:```sh +sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF + + Options FollowSymLinks MultiViews + AddType text/html .php + DirectoryIndex search.php + Require all granted + + +Alias /nominatim $USERHOME/nominatim-project/website +EOFAPACHECONF +#DOCS:``` + +# +# Then enable the configuration and restart apache +# + + sudo a2enconf nominatim +if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: + sudo apache2ctl start #DOCS: +else #DOCS: + sudo systemctl restart apache2 +fi #DOCS: + +# The Nominatim API is now available at `http://localhost/nominatim/`. + +fi #DOCS: + +# +# Option 2: Using nginx +# --------------------- +# +if [ "x$2" == "xinstall-nginx" ]; then #DOCS: + +# Nginx has no native support for php scripts. You need to set up php-fpm for +# this purpose. First install nginx and php-fpm: + + sudo apt install -y nginx php-fpm + +# You need to configure php-fpm to listen on a Unix socket. + +#DOCS:```sh +sudo tee /etc/php/8.1/fpm/pool.d/www.conf << EOF_PHP_FPM_CONF +[www] +; Replace the tcp listener and add the unix socket +listen = /var/run/php8.1-fpm.sock + +; Ensure that the daemon runs as the correct user +listen.owner = www-data +listen.group = www-data +listen.mode = 0666 + +; Unix user of FPM processes +user = www-data +group = www-data + +; Choose process manager type (static, dynamic, ondemand) +pm = ondemand +pm.max_children = 5 +EOF_PHP_FPM_CONF +#DOCS:``` + +# Then create a Nginx configuration to forward http requests to that socket. + +#DOCS:```sh +sudo tee /etc/nginx/sites-available/default << EOF_NGINX_CONF +server { + listen 80 default_server; + listen [::]:80 default_server; + + root $USERHOME/nominatim-project/website; + index search.php index.html; + location / { + try_files \$uri \$uri/ @php; + } + + location @php { + fastcgi_param SCRIPT_FILENAME "\$document_root\$uri.php"; + fastcgi_param PATH_TRANSLATED "\$document_root\$uri.php"; + fastcgi_param QUERY_STRING \$args; + fastcgi_pass unix:/var/run/php8.1-fpm.sock; + fastcgi_index index.php; + include fastcgi_params; + } + + location ~ [^/]\.php(/|$) { + fastcgi_split_path_info ^(.+?\.php)(/.*)$; + if (!-f \$document_root\$fastcgi_script_name) { + return 404; + } + fastcgi_pass unix:/var/run/php7.4-fpm.sock; + fastcgi_index search.php; + include fastcgi.conf; + } +} +EOF_NGINX_CONF +#DOCS:``` + +# If you have some errors, make sure that php8.1-fpm.sock is well under +# /var/run/ and not under /var/run/php. Otherwise change the Nginx configuration +# to /var/run/php/php8.1-fpm.sock. +# +# Enable the configuration and restart Nginx +# + +if [ "x$NOSYSTEMD" == "xyes" ]; then #DOCS: + sudo /usr/sbin/php-fpm8.1 --nodaemonize --fpm-config /etc/php/8.1/fpm/php-fpm.conf & #DOCS: + sudo /usr/sbin/nginx & #DOCS: +else #DOCS: + sudo systemctl restart php8.1-fpm nginx +fi #DOCS: + +# The Nominatim API is now available at `http://localhost/`. + + + +fi #DOCS: From c70ca7f57bb40c2807e0149156be9b305c743321 Mon Sep 17 00:00:00 2001 From: marc tobias Date: Mon, 4 Jul 2022 23:52:36 +0200 Subject: [PATCH 2/2] In tests for PHP 8 disable Just-in-time, it conflicts with tools that determine coverage --- .github/workflows/ci-tests.yml | 1 + test/python/tools/test_exec_utils.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index e3a01aa0..53b76a03 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -70,6 +70,7 @@ jobs: with: php-version: ${{ matrix.php }} tools: phpunit, phpcs, composer + ini-values: opcache.jit=disable - uses: actions/setup-python@v2 with: diff --git a/test/python/tools/test_exec_utils.py b/test/python/tools/test_exec_utils.py index 78650180..26ea92b2 100644 --- a/test/python/tools/test_exec_utils.py +++ b/test/python/tools/test_exec_utils.py @@ -122,7 +122,13 @@ class TestRunApiScript: @staticmethod def test_fail_on_error_output(tmp_path): - (tmp_path / 'website' / 'bad.php').write_text("