Merge pull request #2384 from lonvia/actions-add-icu-tokenizer

CI: run tests on Ubuntu 18

.github/actions/build-nominatim/action.yml

@@ -14,9 +14,9 @@ runs:
           run: |
             sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
             if [ "x$UBUNTUVER" == "x18" ]; then
-                pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium
+                pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium PyYAML==5.1 datrie
             else
-                sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv
+                sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
             fi
           shell: bash
           env:

.github/workflows/ci-tests.yml

@@ -4,16 +4,22 @@ on: [ push, pull_request ]
 
 jobs:
     tests:
-        runs-on: ubuntu-20.04
-
         strategy:
             matrix:
-                postgresql: [9.5, 13]
+                ubuntu: [18, 20]
                 include:
-                    - postgresql: 9.5
+                    - ubuntu: 18
+                      postgresql: 9.5
                       postgis: 2.5
-                    - postgresql: 13
+                      pytest: pytest
+                      php: 7.2
+                    - ubuntu: 20
+                      postgresql: 13
                       postgis: 3
+                      pytest: py.test-3
+                      php: 7.4
+
+        runs-on: ubuntu-${{ matrix.ubuntu }}.04
 
         steps:
             - uses: actions/checkout@v2
@@ -24,9 +30,15 @@ jobs:
             - name: Setup PHP
               uses: shivammathur/setup-php@v2
               with:
-                  php-version: '7.4'
+                  php-version: ${{ matrix.php }}
+                  coverage: xdebug
                   tools: phpunit, phpcs, composer
 
+            - uses: actions/setup-python@v2
+              with:
+                python-version: 3.6
+              if: matrix.ubuntu == 18
+
             - name: Get Date
               id: get-date
               run: |
@@ -43,33 +55,53 @@ jobs:
               with:
                   postgresql-version: ${{ matrix.postgresql }}
                   postgis-version: ${{ matrix.postgis }}
+
             - uses: ./Nominatim/.github/actions/build-nominatim
+              with:
+                  ubuntu: ${{ matrix.ubuntu }}
 
             - name: Install test prerequsites
-              run: sudo apt-get install -y -qq php-codesniffer pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage php-xdebug
+              run: sudo apt-get install -y -qq pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage
+              if: matrix.ubuntu == 20
+
+            - name: Install test prerequsites
+              run: |
+                   pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6
+              if: matrix.ubuntu == 18
 
             - name: PHP linting
               run: phpcs --report-width=120 .
               working-directory: Nominatim
 
             - name: Python linting
-              run: pylint --extension-pkg-whitelist=osmium nominatim
+              run: pylint nominatim
               working-directory: Nominatim
 
             - name: PHP unit tests
               run: phpunit --coverage-clover ../../coverage-php.xml ./
               working-directory: Nominatim/test/php
+              if: matrix.ubuntu == 20
 
             - name: Python unit tests
-              run: py.test-3 --cov=nominatim --cov-report=xml test/python
+              run: $PYTEST --cov=nominatim --cov-report=xml test/python
               working-directory: Nominatim
+              env:
+                PYTEST: ${{ matrix.pytest }}
 
             - name: BDD tests
               run: |
+                  mkdir cov
                   behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 -DPHPCOV=./cov
                   composer require phpunit/phpcov:7.0.2
                   vendor/bin/phpcov merge --clover ../../coverage-bdd.xml ./cov
               working-directory: Nominatim/test/bdd
+              if: matrix.ubuntu == 20
+
+            - name: BDD tests
+              run: |
+                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
+              working-directory: Nominatim/test/bdd
+              if: matrix.ubuntu == 18
 
             - name: BDD tests (legacy_icu tokenizer)
               run: |
@@ -85,6 +117,7 @@ jobs:
                 fail_ci_if_error: false
                 path_to_write_report: ./coverage/codecov_report.txt
                 verbose: true
+              if: matrix.ubuntu == 20
 
     import:
         strategy:

docs/admin/Installation.md

@@ -45,6 +45,7 @@ For running Nominatim:
   * [psutil](https://github.com/giampaolo/psutil)
   * [Jinja2](https://palletsprojects.com/p/jinja/)
   * [PyICU](https://pypi.org/project/PyICU/)
+  * [PyYaml](https://pyyaml.org/) (5.1+)
   * [datrie](https://github.com/pytries/datrie)
   * [PHP](https://php.net) (7.0 or later)
   * PHP-pgsql

docs/develop/Development-Environment.md

@@ -29,7 +29,7 @@ The Nominatim test suite consists of behavioural tests (using behave) and
 unit tests (using PHPUnit for PHP code and pytest for Python code).
 It has the following additional requirements:
 
-* [behave test framework](https://behave.readthedocs.io) >= 1.2.5
+* [behave test framework](https://behave.readthedocs.io) >= 1.2.6
 * [phpunit](https://phpunit.de) >= 7.3
 * [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer)
 * [Pylint](https://pylint.org/) (2.6.0 is used for the CI)

nominatim/config.py

@@ -68,9 +68,9 @@ class Configuration:
         """
         try:
             return int(self.__getattr__(name))
-        except ValueError:
+        except ValueError as exp:
             LOG.fatal("Invalid setting NOMINATIM_%s. Needs to be a number.", name)
-            raise UsageError("Configuration error.")
+            raise UsageError("Configuration error.") from exp
 
 
     def get_libpq_dsn(self):

nominatim/tokenizer/icu_variants.py

@@ -7,12 +7,11 @@ import json
 _ICU_VARIANT_PORPERTY_FIELDS = ['lang']
 
 
-class ICUVariantProperties(namedtuple('_ICUVariantProperties', _ICU_VARIANT_PORPERTY_FIELDS,
+class ICUVariantProperties(namedtuple('_ICUVariantProperties', _ICU_VARIANT_PORPERTY_FIELDS)):
-                                      defaults=(None, )*len(_ICU_VARIANT_PORPERTY_FIELDS))):
     """ Data container for saving properties that describe when a variant
         should be applied.
 
-        Porperty instances are hashable.
+        Property instances are hashable.
     """
     @classmethod
     def from_rules(cls, _):
@@ -52,7 +51,7 @@ def unpickle_variant_set(variant_string):
     """
     data = json.loads(variant_string)
 
-    properties = {int(k): ICUVariantProperties(**v) for k, v in data['properties'].items()}
+    properties = {int(k): ICUVariantProperties.from_rules(v)
-    print(properties)
+                  for k, v in data['properties'].items()}
 
     return set((ICUVariant(src, repl, properties[pid]) for src, repl, pid in data['variants']))

nominatim/tools/exec_utils.py

@@ -70,7 +70,9 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None,
     else:
         cmd = [str(phpcgi_bin)]
 
-    proc = subprocess.run(cmd, cwd=str(project_dir), env=env, capture_output=True,
+    proc = subprocess.run(cmd, cwd=str(project_dir), env=env,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE,
                           check=False)
 
     if proc.returncode != 0 or proc.stderr:

vagrant/Install-on-Centos-7.sh

@@ -42,7 +42,7 @@
                         python3-pip python3-setuptools python3-devel \
                         expat-devel zlib-devel libicu-dev
 
-    pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie
+    pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml
 
 
 #

vagrant/Install-on-Centos-8.sh

@@ -35,7 +35,7 @@
                         python3-pip python3-setuptools python3-devel \
                         expat-devel zlib-devel libicu-dev
 
-    pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie
+    pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml
 
 
 #

vagrant/Install-on-Ubuntu-18.sh

@@ -32,10 +32,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         php php-pgsql php-intl libicu-dev python3-pip \
                         python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
 
-# The python-dotenv adn datrie package that comes with Ubuntu 18.04 is too old, so
+# Some of the Python packages that come with Ubuntu 18.04 are too old, so
 # install the latest version from pip:
 
-    pip3 install python-dotenv datrie
+    pip3 install python-dotenv datrie pyyaml
 
 #
 # System Configuration