osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-12-26 14:36:23 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2384 from lonvia/actions-add-icu-tokenizer

Browse Source 
CI: run tests on Ubuntu 18
This commit is contained in:
Sarah Hoffmann 2021-07-07 14:39:53 +02:00 committed by GitHub
commit a5970d7548
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 59 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/actions/build-nominatim/action.yml vendored
View File

@ -14,9 +14,9 @@ runs:
run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
if [ "x$UBUNTUVER" == "x18" ]; then
pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium
pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium PyYAML==5.1 datrie
else
sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv
sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
fi
shell: bash
env:

51
.github/workflows/ci-tests.yml vendored
View File

@ -4,16 +4,22 @@ on: [ push, pull_request ]
jobs:
tests:
runs-on: ubuntu-20.04
strategy:
matrix:
postgresql: [9.5, 13]
ubuntu: [18, 20]
include:
- postgresql: 9.5
- ubuntu: 18
postgresql: 9.5
postgis: 2.5
- postgresql: 13
pytest: pytest
php: 7.2
- ubuntu: 20
postgresql: 13
postgis: 3
pytest: py.test-3
php: 7.4
runs-on: ubuntu-${{ matrix.ubuntu }}.04
steps:
- uses: actions/checkout@v2
@ -24,9 +30,15 @@ jobs:
- name: Setup PHP
uses: shivammathur/setup-php@v2
with:
php-version: '7.4'
php-version: ${{ matrix.php }}
coverage: xdebug
tools: phpunit, phpcs, composer
- uses: actions/setup-python@v2
with:
python-version: 3.6
if: matrix.ubuntu == 18
- name: Get Date
id: get-date
run: |
@ -43,33 +55,53 @@ jobs:
with:
postgresql-version: ${{ matrix.postgresql }}
postgis-version: ${{ matrix.postgis }}
- uses: ./Nominatim/.github/actions/build-nominatim
with:
ubuntu: ${{ matrix.ubuntu }}
- name: Install test prerequsites
run: sudo apt-get install -y -qq php-codesniffer pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage php-xdebug
run: sudo apt-get install -y -qq pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage
if: matrix.ubuntu == 20
- name: Install test prerequsites
run: |
pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6
if: matrix.ubuntu == 18
- name: PHP linting
run: phpcs --report-width=120 .
working-directory: Nominatim
- name: Python linting
run: pylint --extension-pkg-whitelist=osmium nominatim
run: pylint nominatim
working-directory: Nominatim
- name: PHP unit tests
run: phpunit --coverage-clover ../../coverage-php.xml ./
working-directory: Nominatim/test/php
if: matrix.ubuntu == 20
- name: Python unit tests
run: py.test-3 --cov=nominatim --cov-report=xml test/python
run: $PYTEST --cov=nominatim --cov-report=xml test/python
working-directory: Nominatim
env:
PYTEST: ${{ matrix.pytest }}
- name: BDD tests
run: |
mkdir cov
behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 -DPHPCOV=./cov
composer require phpunit/phpcov:7.0.2
vendor/bin/phpcov merge --clover ../../coverage-bdd.xml ./cov
working-directory: Nominatim/test/bdd
if: matrix.ubuntu == 20
- name: BDD tests
run: |
behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
working-directory: Nominatim/test/bdd
if: matrix.ubuntu == 18
- name: BDD tests (legacy_icu tokenizer)
run: |
@ -85,6 +117,7 @@ jobs:
fail_ci_if_error: false
path_to_write_report: ./coverage/codecov_report.txt
verbose: true
if: matrix.ubuntu == 20
import:
strategy:

1
docs/admin/Installation.md
View File

@ -45,6 +45,7 @@ For running Nominatim:
* [psutil](https://github.com/giampaolo/psutil)
* [Jinja2](https://palletsprojects.com/p/jinja/)
* [PyICU](https://pypi.org/project/PyICU/)
* [PyYaml](https://pyyaml.org/) (5.1+)
* [datrie](https://github.com/pytries/datrie)
* [PHP](https://php.net) (7.0 or later)
* PHP-pgsql

2
docs/develop/Development-Environment.md
View File

@ -29,7 +29,7 @@ The Nominatim test suite consists of behavioural tests (using behave) and
unit tests (using PHPUnit for PHP code and pytest for Python code).
It has the following additional requirements:
* [behave test framework](https://behave.readthedocs.io) >= 1.2.5
* [behave test framework](https://behave.readthedocs.io) >= 1.2.6
* [phpunit](https://phpunit.de) >= 7.3
* [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer)
* [Pylint](https://pylint.org/) (2.6.0 is used for the CI)

4
nominatim/config.py
View File

@ -68,9 +68,9 @@ class Configuration:
"""
try:
return int(self.__getattr__(name))
except ValueError:
except ValueError as exp:
LOG.fatal("Invalid setting NOMINATIM_%s. Needs to be a number.", name)
raise UsageError("Configuration error.")
raise UsageError("Configuration error.") from exp
def get_libpq_dsn(self):

9
nominatim/tokenizer/icu_variants.py
View File

@ -7,12 +7,11 @@ import json
_ICU_VARIANT_PORPERTY_FIELDS = ['lang']
class ICUVariantProperties(namedtuple('_ICUVariantProperties', _ICU_VARIANT_PORPERTY_FIELDS,
defaults=(None, )*len(_ICU_VARIANT_PORPERTY_FIELDS))):
class ICUVariantProperties(namedtuple('_ICUVariantProperties', _ICU_VARIANT_PORPERTY_FIELDS)):
""" Data container for saving properties that describe when a variant
should be applied.
Porperty instances are hashable.
Property instances are hashable.
"""
@classmethod
def from_rules(cls, _):
@ -52,7 +51,7 @@ def unpickle_variant_set(variant_string):
"""
data = json.loads(variant_string)
properties = {int(k): ICUVariantProperties(**v) for k, v in data['properties'].items()}
print(properties)
properties = {int(k): ICUVariantProperties.from_rules(v)
for k, v in data['properties'].items()}
return set((ICUVariant(src, repl, properties[pid]) for src, repl, pid in data['variants']))

4
nominatim/tools/exec_utils.py
View File

@ -70,7 +70,9 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None,
else:
cmd = [str(phpcgi_bin)]
proc = subprocess.run(cmd, cwd=str(project_dir), env=env, capture_output=True,
proc = subprocess.run(cmd, cwd=str(project_dir), env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=False)
if proc.returncode != 0 or proc.stderr:

2
vagrant/Install-on-Centos-7.sh
View File

@ -42,7 +42,7 @@
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel libicu-dev
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml
#

2
vagrant/Install-on-Centos-8.sh
View File

@ -35,7 +35,7 @@
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel libicu-dev
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie
pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml
#

4
vagrant/Install-on-Ubuntu-18.sh
View File

@ -32,10 +32,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
php php-pgsql php-intl libicu-dev python3-pip \
python3-psycopg2 python3-psutil python3-jinja2 python3-icu git
# The python-dotenv adn datrie package that comes with Ubuntu 18.04 is too old, so
# Some of the Python packages that come with Ubuntu 18.04 are too old, so
# install the latest version from pip:
pip3 install python-dotenv datrie
pip3 install python-dotenv datrie pyyaml
#
# System Configuration