diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index a3915616..757decd4 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -14,9 +14,9 @@ runs: run: | sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev if [ "x$UBUNTUVER" == "x18" ]; then - pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium + pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium PyYAML==5.1 datrie else - sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv + sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml fi shell: bash env: diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 8f0ea80d..cea27091 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -4,16 +4,22 @@ on: [ push, pull_request ] jobs: tests: - runs-on: ubuntu-20.04 - strategy: matrix: - postgresql: [9.5, 13] + ubuntu: [18, 20] include: - - postgresql: 9.5 + - ubuntu: 18 + postgresql: 9.5 postgis: 2.5 - - postgresql: 13 + pytest: pytest + php: 7.2 + - ubuntu: 20 + postgresql: 13 postgis: 3 + pytest: py.test-3 + php: 7.4 + + runs-on: ubuntu-${{ matrix.ubuntu }}.04 steps: - uses: actions/checkout@v2 @@ -24,9 +30,15 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: '7.4' + php-version: ${{ matrix.php }} + coverage: xdebug tools: phpunit, phpcs, composer + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + if: matrix.ubuntu == 18 + - name: Get Date id: get-date run: | @@ -43,33 +55,53 @@ jobs: with: postgresql-version: ${{ matrix.postgresql }} postgis-version: ${{ matrix.postgis }} + - uses: ./Nominatim/.github/actions/build-nominatim + with: + ubuntu: ${{ matrix.ubuntu }} - name: Install test prerequsites - run: sudo apt-get install -y -qq php-codesniffer pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage php-xdebug + run: sudo apt-get install -y -qq pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage + if: matrix.ubuntu == 20 + + - name: Install test prerequsites + run: | + pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6 + if: matrix.ubuntu == 18 - name: PHP linting run: phpcs --report-width=120 . working-directory: Nominatim - name: Python linting - run: pylint --extension-pkg-whitelist=osmium nominatim + run: pylint nominatim working-directory: Nominatim - name: PHP unit tests run: phpunit --coverage-clover ../../coverage-php.xml ./ working-directory: Nominatim/test/php + if: matrix.ubuntu == 20 - name: Python unit tests - run: py.test-3 --cov=nominatim --cov-report=xml test/python + run: $PYTEST --cov=nominatim --cov-report=xml test/python working-directory: Nominatim + env: + PYTEST: ${{ matrix.pytest }} - name: BDD tests run: | + mkdir cov behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 -DPHPCOV=./cov composer require phpunit/phpcov:7.0.2 vendor/bin/phpcov merge --clover ../../coverage-bdd.xml ./cov working-directory: Nominatim/test/bdd + if: matrix.ubuntu == 20 + + - name: BDD tests + run: | + behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 + working-directory: Nominatim/test/bdd + if: matrix.ubuntu == 18 - name: BDD tests (legacy_icu tokenizer) run: | @@ -85,6 +117,7 @@ jobs: fail_ci_if_error: false path_to_write_report: ./coverage/codecov_report.txt verbose: true + if: matrix.ubuntu == 20 import: strategy: diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index cc1edf59..76af39c6 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -45,6 +45,7 @@ For running Nominatim: * [psutil](https://github.com/giampaolo/psutil) * [Jinja2](https://palletsprojects.com/p/jinja/) * [PyICU](https://pypi.org/project/PyICU/) + * [PyYaml](https://pyyaml.org/) (5.1+) * [datrie](https://github.com/pytries/datrie) * [PHP](https://php.net) (7.0 or later) * PHP-pgsql diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index 43598b9a..eea69c70 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -29,7 +29,7 @@ The Nominatim test suite consists of behavioural tests (using behave) and unit tests (using PHPUnit for PHP code and pytest for Python code). It has the following additional requirements: -* [behave test framework](https://behave.readthedocs.io) >= 1.2.5 +* [behave test framework](https://behave.readthedocs.io) >= 1.2.6 * [phpunit](https://phpunit.de) >= 7.3 * [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer) * [Pylint](https://pylint.org/) (2.6.0 is used for the CI) diff --git a/nominatim/config.py b/nominatim/config.py index 72aaf0bd..a8436440 100644 --- a/nominatim/config.py +++ b/nominatim/config.py @@ -68,9 +68,9 @@ class Configuration: """ try: return int(self.__getattr__(name)) - except ValueError: + except ValueError as exp: LOG.fatal("Invalid setting NOMINATIM_%s. Needs to be a number.", name) - raise UsageError("Configuration error.") + raise UsageError("Configuration error.") from exp def get_libpq_dsn(self): diff --git a/nominatim/tokenizer/icu_variants.py b/nominatim/tokenizer/icu_variants.py index 5148f3e2..9ebe3684 100644 --- a/nominatim/tokenizer/icu_variants.py +++ b/nominatim/tokenizer/icu_variants.py @@ -7,12 +7,11 @@ import json _ICU_VARIANT_PORPERTY_FIELDS = ['lang'] -class ICUVariantProperties(namedtuple('_ICUVariantProperties', _ICU_VARIANT_PORPERTY_FIELDS, - defaults=(None, )*len(_ICU_VARIANT_PORPERTY_FIELDS))): +class ICUVariantProperties(namedtuple('_ICUVariantProperties', _ICU_VARIANT_PORPERTY_FIELDS)): """ Data container for saving properties that describe when a variant should be applied. - Porperty instances are hashable. + Property instances are hashable. """ @classmethod def from_rules(cls, _): @@ -52,7 +51,7 @@ def unpickle_variant_set(variant_string): """ data = json.loads(variant_string) - properties = {int(k): ICUVariantProperties(**v) for k, v in data['properties'].items()} - print(properties) + properties = {int(k): ICUVariantProperties.from_rules(v) + for k, v in data['properties'].items()} return set((ICUVariant(src, repl, properties[pid]) for src, repl, pid in data['variants'])) diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index 9888d96a..560bb781 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -70,7 +70,9 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None, else: cmd = [str(phpcgi_bin)] - proc = subprocess.run(cmd, cwd=str(project_dir), env=env, capture_output=True, + proc = subprocess.run(cmd, cwd=str(project_dir), env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False) if proc.returncode != 0 or proc.stderr: diff --git a/vagrant/Install-on-Centos-7.sh b/vagrant/Install-on-Centos-7.sh index 9fb90150..d3fd9ef0 100755 --- a/vagrant/Install-on-Centos-7.sh +++ b/vagrant/Install-on-Centos-7.sh @@ -42,7 +42,7 @@ python3-pip python3-setuptools python3-devel \ expat-devel zlib-devel libicu-dev - pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie + pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml # diff --git a/vagrant/Install-on-Centos-8.sh b/vagrant/Install-on-Centos-8.sh index 2330fc3b..a41e846c 100755 --- a/vagrant/Install-on-Centos-8.sh +++ b/vagrant/Install-on-Centos-8.sh @@ -35,7 +35,7 @@ python3-pip python3-setuptools python3-devel \ expat-devel zlib-devel libicu-dev - pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie + pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml # diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh index 63c07bec..dadce086 100755 --- a/vagrant/Install-on-Ubuntu-18.sh +++ b/vagrant/Install-on-Ubuntu-18.sh @@ -32,10 +32,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: php php-pgsql php-intl libicu-dev python3-pip \ python3-psycopg2 python3-psutil python3-jinja2 python3-icu git -# The python-dotenv adn datrie package that comes with Ubuntu 18.04 is too old, so +# Some of the Python packages that come with Ubuntu 18.04 are too old, so # install the latest version from pip: - pip3 install python-dotenv datrie + pip3 install python-dotenv datrie pyyaml # # System Configuration