From 54f295be52b1ddf42849917e626bd171c565442d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 5 Jul 2021 17:15:07 +0200 Subject: [PATCH 1/5] CI: run tests on older Ubuntu version as well --- .github/actions/build-nominatim/action.yml | 4 +-- .github/workflows/ci-tests.yml | 36 +++++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index a3915616..757decd4 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -14,9 +14,9 @@ runs: run: | sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev if [ "x$UBUNTUVER" == "x18" ]; then - pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium + pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium PyYAML==5.1 datrie else - sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv + sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml fi shell: bash env: diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 8f0ea80d..c6792f3a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -4,16 +4,20 @@ on: [ push, pull_request ] jobs: tests: - runs-on: ubuntu-20.04 - strategy: matrix: - postgresql: [9.5, 13] + ubuntu: [18, 20] include: - - postgresql: 9.5 + - ubuntu: 18 + postgresql: 9.5 postgis: 2.5 - - postgresql: 13 + pytest: pytest + - ubuntu: 20 + postgresql: 13 postgis: 3 + pytest: py.test-3 + + runs-on: ubuntu-${{ matrix.ubuntu }}.04 steps: - uses: actions/checkout@v2 @@ -27,6 +31,11 @@ jobs: php-version: '7.4' tools: phpunit, phpcs, composer + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + if: matrix.ubuntu == 18 + - name: Get Date id: get-date run: | @@ -43,17 +52,27 @@ jobs: with: postgresql-version: ${{ matrix.postgresql }} postgis-version: ${{ matrix.postgis }} + - uses: ./Nominatim/.github/actions/build-nominatim + with: + ubuntu: ${{ matrix.ubuntu }} - name: Install test prerequsites run: sudo apt-get install -y -qq php-codesniffer pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage php-xdebug + if: matrix.ubuntu == 20 + + - name: Install test prerequsites + run: | + sudo apt-get install -y -qq php-codesniffer php-codecoverage php-xdebug + pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6 + if: matrix.ubuntu == 18 - name: PHP linting run: phpcs --report-width=120 . working-directory: Nominatim - name: Python linting - run: pylint --extension-pkg-whitelist=osmium nominatim + run: pylint nominatim working-directory: Nominatim - name: PHP unit tests @@ -61,11 +80,14 @@ jobs: working-directory: Nominatim/test/php - name: Python unit tests - run: py.test-3 --cov=nominatim --cov-report=xml test/python + run: $PYTEST --cov=nominatim --cov-report=xml test/python working-directory: Nominatim + env: + PYTEST: ${{ matrix.pytest }} - name: BDD tests run: | + mkdir cov behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 -DPHPCOV=./cov composer require phpunit/phpcov:7.0.2 vendor/bin/phpcov merge --clover ../../coverage-bdd.xml ./cov From 1e86dc1d932a672f1c83b5abc2906016b7500de1 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 6 Jul 2021 09:54:11 +0200 Subject: [PATCH 2/5] remove default parameter for namedtuple This is only available in Python 3.7. --- nominatim/config.py | 4 ++-- nominatim/tokenizer/icu_variants.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/nominatim/config.py b/nominatim/config.py index 72aaf0bd..a8436440 100644 --- a/nominatim/config.py +++ b/nominatim/config.py @@ -68,9 +68,9 @@ class Configuration: """ try: return int(self.__getattr__(name)) - except ValueError: + except ValueError as exp: LOG.fatal("Invalid setting NOMINATIM_%s. Needs to be a number.", name) - raise UsageError("Configuration error.") + raise UsageError("Configuration error.") from exp def get_libpq_dsn(self): diff --git a/nominatim/tokenizer/icu_variants.py b/nominatim/tokenizer/icu_variants.py index 5148f3e2..9ebe3684 100644 --- a/nominatim/tokenizer/icu_variants.py +++ b/nominatim/tokenizer/icu_variants.py @@ -7,12 +7,11 @@ import json _ICU_VARIANT_PORPERTY_FIELDS = ['lang'] -class ICUVariantProperties(namedtuple('_ICUVariantProperties', _ICU_VARIANT_PORPERTY_FIELDS, - defaults=(None, )*len(_ICU_VARIANT_PORPERTY_FIELDS))): +class ICUVariantProperties(namedtuple('_ICUVariantProperties', _ICU_VARIANT_PORPERTY_FIELDS)): """ Data container for saving properties that describe when a variant should be applied. - Porperty instances are hashable. + Property instances are hashable. """ @classmethod def from_rules(cls, _): @@ -52,7 +51,7 @@ def unpickle_variant_set(variant_string): """ data = json.loads(variant_string) - properties = {int(k): ICUVariantProperties(**v) for k, v in data['properties'].items()} - print(properties) + properties = {int(k): ICUVariantProperties.from_rules(v) + for k, v in data['properties'].items()} return set((ICUVariant(src, repl, properties[pid]) for src, repl, pid in data['variants'])) From a2edbbf78a9612193d7751806d310de385549dee Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 6 Jul 2021 16:10:18 +0200 Subject: [PATCH 3/5] cannot use capture_output in subprocess.run Only available since Python 3.7. --- nominatim/tools/exec_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nominatim/tools/exec_utils.py b/nominatim/tools/exec_utils.py index 9888d96a..560bb781 100644 --- a/nominatim/tools/exec_utils.py +++ b/nominatim/tools/exec_utils.py @@ -70,7 +70,9 @@ def run_api_script(endpoint, project_dir, extra_env=None, phpcgi_bin=None, else: cmd = [str(phpcgi_bin)] - proc = subprocess.run(cmd, cwd=str(project_dir), env=env, capture_output=True, + proc = subprocess.run(cmd, cwd=str(project_dir), env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False) if proc.returncode != 0 or proc.stderr: From 42e08da7ca8d93718237b1cfdb348b294d70aa52 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 6 Jul 2021 22:52:57 +0200 Subject: [PATCH 4/5] enable PHP 7.2 for Ubuntu 18 CI --- .github/workflows/ci-tests.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index c6792f3a..cea27091 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -12,10 +12,12 @@ jobs: postgresql: 9.5 postgis: 2.5 pytest: pytest + php: 7.2 - ubuntu: 20 postgresql: 13 postgis: 3 pytest: py.test-3 + php: 7.4 runs-on: ubuntu-${{ matrix.ubuntu }}.04 @@ -28,7 +30,8 @@ jobs: - name: Setup PHP uses: shivammathur/setup-php@v2 with: - php-version: '7.4' + php-version: ${{ matrix.php }} + coverage: xdebug tools: phpunit, phpcs, composer - uses: actions/setup-python@v2 @@ -58,12 +61,11 @@ jobs: ubuntu: ${{ matrix.ubuntu }} - name: Install test prerequsites - run: sudo apt-get install -y -qq php-codesniffer pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage php-xdebug + run: sudo apt-get install -y -qq pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage if: matrix.ubuntu == 20 - name: Install test prerequsites run: | - sudo apt-get install -y -qq php-codesniffer php-codecoverage php-xdebug pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6 if: matrix.ubuntu == 18 @@ -78,6 +80,7 @@ jobs: - name: PHP unit tests run: phpunit --coverage-clover ../../coverage-php.xml ./ working-directory: Nominatim/test/php + if: matrix.ubuntu == 20 - name: Python unit tests run: $PYTEST --cov=nominatim --cov-report=xml test/python @@ -92,6 +95,13 @@ jobs: composer require phpunit/phpcov:7.0.2 vendor/bin/phpcov merge --clover ../../coverage-bdd.xml ./cov working-directory: Nominatim/test/bdd + if: matrix.ubuntu == 20 + + - name: BDD tests + run: | + behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 + working-directory: Nominatim/test/bdd + if: matrix.ubuntu == 18 - name: BDD tests (legacy_icu tokenizer) run: | @@ -107,6 +117,7 @@ jobs: fail_ci_if_error: false path_to_write_report: ./coverage/codecov_report.txt verbose: true + if: matrix.ubuntu == 20 import: strategy: From c216144dd1cbc6ee19834b19329c55e2d74d9a73 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 6 Jul 2021 23:04:01 +0200 Subject: [PATCH 5/5] add missing pyyaml requirement --- docs/admin/Installation.md | 1 + docs/develop/Development-Environment.md | 2 +- vagrant/Install-on-Centos-7.sh | 2 +- vagrant/Install-on-Centos-8.sh | 2 +- vagrant/Install-on-Ubuntu-18.sh | 4 ++-- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index cc1edf59..76af39c6 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -45,6 +45,7 @@ For running Nominatim: * [psutil](https://github.com/giampaolo/psutil) * [Jinja2](https://palletsprojects.com/p/jinja/) * [PyICU](https://pypi.org/project/PyICU/) + * [PyYaml](https://pyyaml.org/) (5.1+) * [datrie](https://github.com/pytries/datrie) * [PHP](https://php.net) (7.0 or later) * PHP-pgsql diff --git a/docs/develop/Development-Environment.md b/docs/develop/Development-Environment.md index 43598b9a..eea69c70 100644 --- a/docs/develop/Development-Environment.md +++ b/docs/develop/Development-Environment.md @@ -29,7 +29,7 @@ The Nominatim test suite consists of behavioural tests (using behave) and unit tests (using PHPUnit for PHP code and pytest for Python code). It has the following additional requirements: -* [behave test framework](https://behave.readthedocs.io) >= 1.2.5 +* [behave test framework](https://behave.readthedocs.io) >= 1.2.6 * [phpunit](https://phpunit.de) >= 7.3 * [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer) * [Pylint](https://pylint.org/) (2.6.0 is used for the CI) diff --git a/vagrant/Install-on-Centos-7.sh b/vagrant/Install-on-Centos-7.sh index 9fb90150..d3fd9ef0 100755 --- a/vagrant/Install-on-Centos-7.sh +++ b/vagrant/Install-on-Centos-7.sh @@ -42,7 +42,7 @@ python3-pip python3-setuptools python3-devel \ expat-devel zlib-devel libicu-dev - pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie + pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml # diff --git a/vagrant/Install-on-Centos-8.sh b/vagrant/Install-on-Centos-8.sh index 2330fc3b..a41e846c 100755 --- a/vagrant/Install-on-Centos-8.sh +++ b/vagrant/Install-on-Centos-8.sh @@ -35,7 +35,7 @@ python3-pip python3-setuptools python3-devel \ expat-devel zlib-devel libicu-dev - pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie + pip3 install --user psycopg2 python-dotenv psutil Jinja2 PyICU datrie pyyaml # diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh index 63c07bec..dadce086 100755 --- a/vagrant/Install-on-Ubuntu-18.sh +++ b/vagrant/Install-on-Ubuntu-18.sh @@ -32,10 +32,10 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: php php-pgsql php-intl libicu-dev python3-pip \ python3-psycopg2 python3-psutil python3-jinja2 python3-icu git -# The python-dotenv adn datrie package that comes with Ubuntu 18.04 is too old, so +# Some of the Python packages that come with Ubuntu 18.04 are too old, so # install the latest version from pip: - pip3 install python-dotenv datrie + pip3 install python-dotenv datrie pyyaml # # System Configuration