osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-12-25 14:02:12 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2341 from lonvia/cleanup-python-tests

Browse Source 
Cleanup and linting of python tests
This commit is contained in:
Sarah Hoffmann 2021-05-20 17:30:30 +02:00 committed by GitHub
commit 8bf15fa691
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 1067 additions and 1089 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.pylintrc
View File

@ -12,4 +12,4 @@ ignored-modules=icu
ignored-classes=NominatimArgs,closing ignored-classes=NominatimArgs,closing
disable=too-few-public-methods,duplicate-code disable=too-few-public-methods,duplicate-code
good-names=i,x,y,fd good-names=i,x,y,fd,db

118
test/python/conftest.py
View File

@ -1,12 +1,9 @@
import importlib
import itertools import itertools
import sys import sys
from pathlib import Path from pathlib import Path
import psycopg2 import psycopg2
import psycopg2.extras
import pytest import pytest
import tempfile
SRC_DIR = Path(__file__) / '..' / '..' / '..' SRC_DIR = Path(__file__) / '..' / '..' / '..'
@ -16,43 +13,12 @@ sys.path.insert(0, str(SRC_DIR.resolve()))
from nominatim.config import Configuration from nominatim.config import Configuration
from nominatim.db import connection from nominatim.db import connection
from nominatim.db.sql_preprocessor import SQLPreprocessor from nominatim.db.sql_preprocessor import SQLPreprocessor
from nominatim.db import properties
import nominatim.tokenizer.factory import nominatim.tokenizer.factory
import nominatim.cli
import dummy_tokenizer import dummy_tokenizer
import mocks import mocks
from cursor import CursorForTesting
class _TestingCursor(psycopg2.extras.DictCursor):
""" Extension to the DictCursor class that provides execution
short-cuts that simplify writing assertions.
"""
def scalar(self, sql, params=None):
""" Execute a query with a single return value and return this value.
Raises an assertion when not exactly one row is returned.
"""
self.execute(sql, params)
assert self.rowcount == 1
return self.fetchone()[0]
def row_set(self, sql, params=None):
""" Execute a query and return the result as a set of tuples.
"""
self.execute(sql, params)
return set((tuple(row) for row in self))
def table_exists(self, table):
""" Check that a table with the given name exists in the database.
"""
num = self.scalar("""SELECT count(*) FROM pg_tables
WHERE tablename = %s""", (table, ))
return num == 1
def table_rows(self, table):
""" Return the number of rows in the given table.
"""
return self.scalar('SELECT count(*) FROM ' + table)
@pytest.fixture @pytest.fixture
@ -70,7 +36,7 @@ def temp_db(monkeypatch):
conn.close() conn.close()
monkeypatch.setenv('NOMINATIM_DATABASE_DSN' , 'dbname=' + name) monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'dbname=' + name)
yield name yield name
@ -113,18 +79,20 @@ def temp_db_cursor(temp_db):
""" """
conn = psycopg2.connect('dbname=' + temp_db) conn = psycopg2.connect('dbname=' + temp_db)
conn.set_isolation_level(0) conn.set_isolation_level(0)
with conn.cursor(cursor_factory=_TestingCursor) as cur: with conn.cursor(cursor_factory=CursorForTesting) as cur:
yield cur yield cur
conn.close() conn.close()
@pytest.fixture @pytest.fixture
def table_factory(temp_db_cursor): def table_factory(temp_db_cursor):
""" A fixture that creates new SQL tables, potentially filled with
content.
"""
def mk_table(name, definition='id INT', content=None): def mk_table(name, definition='id INT', content=None):
temp_db_cursor.execute('CREATE TABLE {} ({})'.format(name, definition)) temp_db_cursor.execute('CREATE TABLE {} ({})'.format(name, definition))
if content is not None: if content is not None:
psycopg2.extras.execute_values( temp_db_cursor.execute_values("INSERT INTO {} VALUES %s".format(name), content)
temp_db_cursor, "INSERT INTO {} VALUES %s".format(name), content)
return mk_table return mk_table
@ -138,51 +106,58 @@ def def_config():
data=SRC_DIR / 'data') data=SRC_DIR / 'data')
return cfg return cfg
@pytest.fixture @pytest.fixture
def src_dir(): def src_dir():
return SRC_DIR.resolve() return SRC_DIR.resolve()
@pytest.fixture
def tmp_phplib_dir():
with tempfile.TemporaryDirectory() as phpdir:
(Path(phpdir) / 'admin').mkdir()
yield Path(phpdir) @pytest.fixture
def cli_call():
def _call_nominatim(*args):
return nominatim.cli.nominatim(module_dir='MODULE NOT AVAILABLE',
osm2pgsql_path='OSM2PGSQL NOT AVAILABLE',
phplib_dir=str(SRC_DIR / 'lib-php'),
data_dir=str(SRC_DIR / 'data'),
phpcgi_path='/usr/bin/php-cgi',
sqllib_dir=str(SRC_DIR / 'lib-sql'),
config_dir=str(SRC_DIR / 'settings'),
cli_args=args)
return _call_nominatim
@pytest.fixture @pytest.fixture
def property_table(table_factory): def property_table(table_factory, temp_db_conn):
table_factory('nominatim_properties', 'property TEXT, value TEXT') table_factory('nominatim_properties', 'property TEXT, value TEXT')
return mocks.MockPropertyTable(temp_db_conn)
@pytest.fixture @pytest.fixture
def status_table(temp_db_conn): def status_table(table_factory):
""" Create an empty version of the status table and """ Create an empty version of the status table and
the status logging table. the status logging table.
""" """
with temp_db_conn.cursor() as cur: table_factory('import_status',
cur.execute("""CREATE TABLE import_status ( """lastimportdate timestamp with time zone NOT NULL,
lastimportdate timestamp with time zone NOT NULL,
sequence_id integer, sequence_id integer,
indexed boolean indexed boolean""")
)""") table_factory('import_osmosis_log',
cur.execute("""CREATE TABLE import_osmosis_log ( """batchend timestamp,
batchend timestamp,
batchseq integer, batchseq integer,
batchsize bigint, batchsize bigint,
starttime timestamp, starttime timestamp,
endtime timestamp, endtime timestamp,
event text event text""")
)""")
temp_db_conn.commit()
@pytest.fixture @pytest.fixture
def place_table(temp_db_with_extensions, temp_db_conn): def place_table(temp_db_with_extensions, table_factory):
""" Create an empty version of the place table. """ Create an empty version of the place table.
""" """
with temp_db_conn.cursor() as cur: table_factory('place',
cur.execute("""CREATE TABLE place ( """osm_id int8 NOT NULL,
osm_id int8 NOT NULL,
osm_type char(1) NOT NULL, osm_type char(1) NOT NULL,
class text NOT NULL, class text NOT NULL,
type text NOT NULL, type text NOT NULL,
@ -190,8 +165,7 @@ def place_table(temp_db_with_extensions, temp_db_conn):
admin_level smallint, admin_level smallint,
address hstore, address hstore,
extratags hstore, extratags hstore,
geometry Geometry(Geometry,4326) NOT NULL)""") geometry Geometry(Geometry,4326) NOT NULL""")
temp_db_conn.commit()
@pytest.fixture @pytest.fixture
@ -217,10 +191,9 @@ def placex_table(temp_db_with_extensions, temp_db_conn):
@pytest.fixture @pytest.fixture
def osmline_table(temp_db_with_extensions, temp_db_conn): def osmline_table(temp_db_with_extensions, table_factory):
with temp_db_conn.cursor() as cur: table_factory('location_property_osmline',
cur.execute("""CREATE TABLE location_property_osmline ( """place_id BIGINT,
place_id BIGINT,
osm_id BIGINT, osm_id BIGINT,
parent_place_id BIGINT, parent_place_id BIGINT,
geometry_sector INTEGER, geometry_sector INTEGER,
@ -233,8 +206,7 @@ def osmline_table(temp_db_with_extensions, temp_db_conn):
interpolationtype TEXT, interpolationtype TEXT,
address HSTORE, address HSTORE,
postcode TEXT, postcode TEXT,
country_code VARCHAR(2))""") country_code VARCHAR(2)""")
temp_db_conn.commit()
@pytest.fixture @pytest.fixture
@ -254,7 +226,7 @@ def osm2pgsql_options(temp_db):
main_data='', main_index='')) main_data='', main_index=''))
@pytest.fixture @pytest.fixture
def sql_preprocessor(temp_db_conn, tmp_path, monkeypatch, table_factory): def sql_preprocessor(temp_db_conn, tmp_path, table_factory):
table_factory('country_name', 'partition INT', ((0, ), (1, ), (2, ))) table_factory('country_name', 'partition INT', ((0, ), (1, ), (2, )))
cfg = Configuration(None, SRC_DIR.resolve() / 'settings') cfg = Configuration(None, SRC_DIR.resolve() / 'settings')
cfg.set_libdirs(module='.', osm2pgsql='.', php=SRC_DIR / 'lib-php', cfg.set_libdirs(module='.', osm2pgsql='.', php=SRC_DIR / 'lib-php',
@ -264,18 +236,18 @@ def sql_preprocessor(temp_db_conn, tmp_path, monkeypatch, table_factory):
@pytest.fixture @pytest.fixture
def tokenizer_mock(monkeypatch, property_table, temp_db_conn, tmp_path): def tokenizer_mock(monkeypatch, property_table):
""" Sets up the configuration so that the test dummy tokenizer will be """ Sets up the configuration so that the test dummy tokenizer will be
loaded when the tokenizer factory is used. Also returns a factory loaded when the tokenizer factory is used. Also returns a factory
with which a new dummy tokenizer may be created. with which a new dummy tokenizer may be created.
""" """
monkeypatch.setenv('NOMINATIM_TOKENIZER', 'dummy') monkeypatch.setenv('NOMINATIM_TOKENIZER', 'dummy')
def _import_dummy(module, *args, **kwargs): def _import_dummy(*args, **kwargs):
return dummy_tokenizer return dummy_tokenizer
monkeypatch.setattr(nominatim.tokenizer.factory, "_import_tokenizer", _import_dummy) monkeypatch.setattr(nominatim.tokenizer.factory, "_import_tokenizer", _import_dummy)
properties.set_property(temp_db_conn, 'tokenizer', 'dummy') property_table.set('tokenizer', 'dummy')
def _create_tokenizer(): def _create_tokenizer():
return dummy_tokenizer.DummyTokenizer(None, None) return dummy_tokenizer.DummyTokenizer(None, None)

52
test/python/cursor.py Normal file
View File

@ -0,0 +1,52 @@
"""
Specialised psycopg2 cursor with shortcut functions useful for testing.
"""
import psycopg2.extras
class CursorForTesting(psycopg2.extras.DictCursor):
""" Extension to the DictCursor class that provides execution
short-cuts that simplify writing assertions.
"""
def scalar(self, sql, params=None):
""" Execute a query with a single return value and return this value.
Raises an assertion when not exactly one row is returned.
"""
self.execute(sql, params)
assert self.rowcount == 1
return self.fetchone()[0]
def row_set(self, sql, params=None):
""" Execute a query and return the result as a set of tuples.
Fails when the SQL command returns duplicate rows.
"""
self.execute(sql, params)
result = set((tuple(row) for row in self))
assert len(result) == self.rowcount
return result
def table_exists(self, table):
""" Check that a table with the given name exists in the database.
"""
num = self.scalar("""SELECT count(*) FROM pg_tables
WHERE tablename = %s""", (table, ))
return num == 1
def table_rows(self, table, where=None):
""" Return the number of rows in the given table.
"""
if where is None:
return self.scalar('SELECT count(*) FROM ' + table)
return self.scalar('SELECT count(*) FROM {} WHERE {}'.format(table, where))
def execute_values(self, *args, **kwargs):
""" Execute the execute_values() function on the cursor.
"""
psycopg2.extras.execute_values(self, *args, **kwargs)

16
test/python/dummy_tokenizer.py
View File

@ -17,16 +17,17 @@ class DummyTokenizer:
def init_new_db(self, *args, **kwargs): def init_new_db(self, *args, **kwargs):
assert self.init_state == None assert self.init_state is None
self.init_state = "new" self.init_state = "new"
def init_from_project(self): def init_from_project(self):
assert self.init_state == None assert self.init_state is None
self.init_state = "loaded" self.init_state = "loaded"
def finalize_import(self, _): @staticmethod
def finalize_import(_):
pass pass
@ -51,10 +52,12 @@ class DummyNameAnalyzer:
def close(self): def close(self):
pass pass
def normalize_postcode(self, postcode): @staticmethod
def normalize_postcode(postcode):
return postcode return postcode
def update_postcodes_from_db(self): @staticmethod
def update_postcodes_from_db():
pass pass
def update_special_phrases(self, phrases, should_replace): def update_special_phrases(self, phrases, should_replace):
@ -63,5 +66,6 @@ class DummyNameAnalyzer:
def add_country_names(self, code, names): def add_country_names(self, code, names):
self.analyser_cache['countries'].append((code, names)) self.analyser_cache['countries'].append((code, names))
def process_place(self, place): @staticmethod
def process_place(place):
return {} return {}

21
test/python/mocks.py
View File

@ -5,6 +5,8 @@ import itertools
import psycopg2.extras import psycopg2.extras
from nominatim.db import properties
class MockParamCapture: class MockParamCapture:
""" Mock that records the parameters with which a function was called """ Mock that records the parameters with which a function was called
as well as the number of calls. as well as the number of calls.
@ -12,6 +14,8 @@ class MockParamCapture:
def __init__(self, retval=0): def __init__(self, retval=0):
self.called = 0 self.called = 0
self.return_value = retval self.return_value = retval
self.last_args = None
self.last_kwargs = None
def __call__(self, *args, **kwargs): def __call__(self, *args, **kwargs):
self.called += 1 self.called += 1
@ -37,11 +41,11 @@ class MockWordTable:
conn.commit() conn.commit()
def add_special(self, word_token, word, cls, typ, op): def add_special(self, word_token, word, cls, typ, oper):
with self.conn.cursor() as cur: with self.conn.cursor() as cur:
cur.execute("""INSERT INTO word (word_token, word, class, type, operator) cur.execute("""INSERT INTO word (word_token, word, class, type, operator)
VALUES (%s, %s, %s, %s, %s) VALUES (%s, %s, %s, %s, %s)
""", (word_token, word, cls, typ, op)) """, (word_token, word, cls, typ, oper))
self.conn.commit() self.conn.commit()
@ -125,3 +129,16 @@ class MockPlacexTable:
admin_level, address, extratags, 'SRID=4326;' + geom, admin_level, address, extratags, 'SRID=4326;' + geom,
country)) country))
self.conn.commit() self.conn.commit()
class MockPropertyTable:
""" A property table for testing.
"""
def __init__(self, conn):
self.conn = conn
def set(self, name, value):
""" Set a property in the table to the given value.
"""
properties.set_property(self.conn, name, value)

276
test/python/test_cli.py
View File

@ -5,8 +5,6 @@ These tests just check that the various command line parameters route to the
correct functionionality. They use a lot of monkeypatching to avoid executing correct functionionality. They use a lot of monkeypatching to avoid executing
the actual functions. the actual functions.
""" """
from pathlib import Path
import pytest import pytest
import nominatim.db.properties import nominatim.db.properties
@ -26,19 +24,6 @@ import nominatim.tokenizer.factory
from mocks import MockParamCapture from mocks import MockParamCapture
SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()
def call_nominatim(*args):
return nominatim.cli.nominatim(module_dir='build/module',
osm2pgsql_path='build/osm2pgsql/osm2pgsql',
phplib_dir=str(SRC_DIR / 'lib-php'),
data_dir=str(SRC_DIR / 'data'),
phpcgi_path='/usr/bin/php-cgi',
sqllib_dir=str(SRC_DIR / 'lib-sql'),
config_dir=str(SRC_DIR / 'settings'),
cli_args=args)
@pytest.fixture @pytest.fixture
def mock_run_legacy(monkeypatch): def mock_run_legacy(monkeypatch):
mock = MockParamCapture() mock = MockParamCapture()
@ -57,8 +42,96 @@ def mock_func_factory(monkeypatch):
return get_mock return get_mock
@pytest.fixture
def tokenizer_mock(monkeypatch): class TestCli:
@pytest.fixture(autouse=True)
def setup_cli_call(self, cli_call):
self.call_nominatim = cli_call
def test_cli_help(self, capsys):
""" Running nominatim tool without arguments prints help.
"""
assert self.call_nominatim() == 1
captured = capsys.readouterr()
assert captured.out.startswith('usage:')
@pytest.mark.parametrize("command,script", [
(('add-data', '--file', 'foo.osm'), 'update'),
(('export',), 'export')
])
def test_legacy_commands_simple(self, mock_run_legacy, command, script):
assert self.call_nominatim(*command) == 0
assert mock_run_legacy.called == 1
assert mock_run_legacy.last_args[0] == script + '.php'
@pytest.mark.parametrize("params", [('--warm', ),
('--warm', '--reverse-only'),
('--warm', '--search-only')])
def test_admin_command_legacy(self, mock_func_factory, params):
mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script')
assert self.call_nominatim('admin', *params) == 0
assert mock_run_legacy.called == 1
def test_admin_command_check_database(self, mock_func_factory):
mock = mock_func_factory(nominatim.tools.check_database, 'check_database')
assert self.call_nominatim('admin', '--check-database') == 0
assert mock.called == 1
@pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc'),
('node', 12), ('way', 8), ('relation', 32)])
def test_add_data_command(self, mock_run_legacy, name, oid):
assert self.call_nominatim('add-data', '--' + name, str(oid)) == 0
assert mock_run_legacy.called == 1
assert mock_run_legacy.last_args == ('update.php', '--import-' + name, oid)
def test_serve_command(self, mock_func_factory):
func = mock_func_factory(nominatim.cli, 'run_php_server')
self.call_nominatim('serve')
assert func.called == 1
@pytest.mark.parametrize("params", [('search', '--query', 'new'),
('reverse', '--lat', '0', '--lon', '0'),
('lookup', '--id', 'N1'),
('details', '--node', '1'),
('details', '--way', '1'),
('details', '--relation', '1'),
('details', '--place_id', '10001'),
('status',)])
def test_api_commands_simple(self, mock_func_factory, params):
mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
assert self.call_nominatim(*params) == 0
assert mock_run_api.called == 1
assert mock_run_api.last_args[0] == params[0]
class TestCliWithDb:
@pytest.fixture(autouse=True)
def setup_cli_call(self, cli_call, temp_db):
self.call_nominatim = cli_call
@pytest.fixture(autouse=True)
def setup_tokenizer_mock(self, monkeypatch):
class DummyTokenizer: class DummyTokenizer:
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.update_sql_functions_called = False self.update_sql_functions_called = False
@ -71,43 +144,23 @@ def tokenizer_mock(monkeypatch):
self.finalize_import_called = True self.finalize_import_called = True
tok = DummyTokenizer() tok = DummyTokenizer()
monkeypatch.setattr(nominatim.tokenizer.factory, 'get_tokenizer_for_db' , monkeypatch.setattr(nominatim.tokenizer.factory, 'get_tokenizer_for_db',
lambda *args: tok) lambda *args: tok)
monkeypatch.setattr(nominatim.tokenizer.factory, 'create_tokenizer' , monkeypatch.setattr(nominatim.tokenizer.factory, 'create_tokenizer',
lambda *args: tok) lambda *args: tok)
return tok self.tokenizer_mock = tok
def test_cli_help(capsys): def test_import_missing_file(self):
""" Running nominatim tool without arguments prints help. assert self.call_nominatim('import', '--osm-file', 'sfsafegwedgw.reh.erh') == 1
"""
assert 1 == call_nominatim()
captured = capsys.readouterr()
assert captured.out.startswith('usage:')
@pytest.mark.parametrize("command,script", [ def test_import_bad_file(self):
(('add-data', '--file', 'foo.osm'), 'update'), assert self.call_nominatim('import', '--osm-file', '.') == 1
(('export',), 'export')
])
def test_legacy_commands_simple(mock_run_legacy, command, script):
assert 0 == call_nominatim(*command)
assert mock_run_legacy.called == 1
assert mock_run_legacy.last_args[0] == script + '.php'
def test_import_missing_file(temp_db): def test_import_full(self, mock_func_factory):
assert 1 == call_nominatim('import', '--osm-file', 'sfsafegweweggdgw.reh.erh')
def test_import_bad_file(temp_db):
assert 1 == call_nominatim('import', '--osm-file', '.')
def test_import_full(temp_db, mock_func_factory, tokenizer_mock):
mocks = [ mocks = [
mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'), mock_func_factory(nominatim.tools.database_import, 'setup_database_skeleton'),
mock_func_factory(nominatim.tools.database_import, 'import_osm_data'), mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
@ -128,8 +181,8 @@ def test_import_full(temp_db, mock_func_factory, tokenizer_mock):
cf_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions') cf_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
assert 0 == call_nominatim('import', '--osm-file', __file__) assert self.call_nominatim('import', '--osm-file', __file__) == 0
assert tokenizer_mock.finalize_import_called assert self.tokenizer_mock.finalize_import_called
assert cf_mock.called > 1 assert cf_mock.called > 1
@ -137,7 +190,7 @@ def test_import_full(temp_db, mock_func_factory, tokenizer_mock):
assert mock.called == 1, "Mock '{}' not called".format(mock.func_name) assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
def test_import_continue_load_data(temp_db, mock_func_factory, tokenizer_mock): def test_import_continue_load_data(self, mock_func_factory):
mocks = [ mocks = [
mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'), mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
mock_func_factory(nominatim.tools.database_import, 'load_data'), mock_func_factory(nominatim.tools.database_import, 'load_data'),
@ -149,15 +202,15 @@ def test_import_continue_load_data(temp_db, mock_func_factory, tokenizer_mock):
mock_func_factory(nominatim.db.properties, 'set_property') mock_func_factory(nominatim.db.properties, 'set_property')
] ]
assert 0 == call_nominatim('import', '--continue', 'load-data') assert self.call_nominatim('import', '--continue', 'load-data') == 0
assert tokenizer_mock.finalize_import_called assert self.tokenizer_mock.finalize_import_called
for mock in mocks: for mock in mocks:
assert mock.called == 1, "Mock '{}' not called".format(mock.func_name) assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
def test_import_continue_indexing(temp_db, mock_func_factory, placex_table, def test_import_continue_indexing(self, mock_func_factory, placex_table,
temp_db_conn, tokenizer_mock): temp_db_conn):
mocks = [ mocks = [
mock_func_factory(nominatim.tools.database_import, 'create_search_indices'), mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim.tools.database_import, 'create_country_names'), mock_func_factory(nominatim.tools.database_import, 'create_country_names'),
@ -166,7 +219,7 @@ def test_import_continue_indexing(temp_db, mock_func_factory, placex_table,
mock_func_factory(nominatim.db.properties, 'set_property') mock_func_factory(nominatim.db.properties, 'set_property')
] ]
assert 0 == call_nominatim('import', '--continue', 'indexing') assert self.call_nominatim('import', '--continue', 'indexing') == 0
for mock in mocks: for mock in mocks:
assert mock.called == 1, "Mock '{}' not called".format(mock.func_name) assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
@ -174,11 +227,11 @@ def test_import_continue_indexing(temp_db, mock_func_factory, placex_table,
assert temp_db_conn.index_exists('idx_placex_pendingsector') assert temp_db_conn.index_exists('idx_placex_pendingsector')
# Calling it again still works for the index # Calling it again still works for the index
assert 0 == call_nominatim('import', '--continue', 'indexing') assert self.call_nominatim('import', '--continue', 'indexing') == 0
assert temp_db_conn.index_exists('idx_placex_pendingsector') assert temp_db_conn.index_exists('idx_placex_pendingsector')
def test_import_continue_postprocess(temp_db, mock_func_factory, tokenizer_mock): def test_import_continue_postprocess(self, mock_func_factory):
mocks = [ mocks = [
mock_func_factory(nominatim.tools.database_import, 'create_search_indices'), mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim.tools.database_import, 'create_country_names'), mock_func_factory(nominatim.tools.database_import, 'create_country_names'),
@ -186,161 +239,110 @@ def test_import_continue_postprocess(temp_db, mock_func_factory, tokenizer_mock)
mock_func_factory(nominatim.db.properties, 'set_property') mock_func_factory(nominatim.db.properties, 'set_property')
] ]
assert 0 == call_nominatim('import', '--continue', 'db-postprocess') assert self.call_nominatim('import', '--continue', 'db-postprocess') == 0
assert tokenizer_mock.finalize_import_called assert self.tokenizer_mock.finalize_import_called
for mock in mocks: for mock in mocks:
assert mock.called == 1, "Mock '{}' not called".format(mock.func_name) assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
def test_freeze_command(mock_func_factory, temp_db): def test_freeze_command(self, mock_func_factory):
mock_drop = mock_func_factory(nominatim.tools.freeze, 'drop_update_tables') mock_drop = mock_func_factory(nominatim.tools.freeze, 'drop_update_tables')
mock_flatnode = mock_func_factory(nominatim.tools.freeze, 'drop_flatnode_file') mock_flatnode = mock_func_factory(nominatim.tools.freeze, 'drop_flatnode_file')
assert 0 == call_nominatim('freeze') assert self.call_nominatim('freeze') == 0
assert mock_drop.called == 1 assert mock_drop.called == 1
assert mock_flatnode.called == 1 assert mock_flatnode.called == 1
@pytest.mark.parametrize("params", [('--warm', ),
('--warm', '--reverse-only'),
('--warm', '--search-only')])
def test_admin_command_legacy(mock_func_factory, params):
mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script')
assert 0 == call_nominatim('admin', *params) @pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))])
def test_admin_command_tool(self, mock_func_factory, func, params):
assert mock_run_legacy.called == 1
@pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))])
def test_admin_command_tool(temp_db, mock_func_factory, func, params):
mock = mock_func_factory(nominatim.tools.admin, func) mock = mock_func_factory(nominatim.tools.admin, func)
assert 0 == call_nominatim('admin', *params) assert self.call_nominatim('admin', *params) == 0
assert mock.called == 1 assert mock.called == 1
def test_admin_command_check_database(mock_func_factory): @pytest.mark.parametrize("params,do_bnds,do_ranks", [
mock = mock_func_factory(nominatim.tools.check_database, 'check_database')
assert 0 == call_nominatim('admin', '--check-database')
assert mock.called == 1
@pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc'),
('node', 12), ('way', 8), ('relation', 32)])
def test_add_data_command(mock_run_legacy, name, oid):
assert 0 == call_nominatim('add-data', '--' + name, str(oid))
assert mock_run_legacy.called == 1
assert mock_run_legacy.last_args == ('update.php', '--import-' + name, oid)
@pytest.mark.parametrize("params,do_bnds,do_ranks", [
([], 1, 1), ([], 1, 1),
(['--boundaries-only'], 1, 0), (['--boundaries-only'], 1, 0),
(['--no-boundaries'], 0, 1), (['--no-boundaries'], 0, 1),
(['--boundaries-only', '--no-boundaries'], 0, 0)]) (['--boundaries-only', '--no-boundaries'], 0, 0)])
def test_index_command(mock_func_factory, temp_db_cursor, tokenizer_mock, def test_index_command(self, mock_func_factory, table_factory,
params, do_bnds, do_ranks): params, do_bnds, do_ranks):
temp_db_cursor.execute("CREATE TABLE import_status (indexed bool)") table_factory('import_status', 'indexed bool')
bnd_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_boundaries') bnd_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_boundaries')
rank_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_by_rank') rank_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_by_rank')
assert 0 == call_nominatim('index', *params) assert self.call_nominatim('index', *params) == 0
assert bnd_mock.called == do_bnds assert bnd_mock.called == do_bnds
assert rank_mock.called == do_ranks assert rank_mock.called == do_ranks
@pytest.mark.parametrize("no_replace", [(True), (False)]) @pytest.mark.parametrize("no_replace", [(True), (False)])
def test_special_phrases_wiki_command(temp_db, mock_func_factory, tokenizer_mock, no_replace): def test_special_phrases_wiki_command(self, mock_func_factory, no_replace):
func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases') func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
if no_replace: if no_replace:
call_nominatim('special-phrases', '--import-from-wiki', '--no-replace') self.call_nominatim('special-phrases', '--import-from-wiki', '--no-replace')
else: else:
call_nominatim('special-phrases', '--import-from-wiki') self.call_nominatim('special-phrases', '--import-from-wiki')
assert func.called == 1 assert func.called == 1
@pytest.mark.parametrize("no_replace", [(True), (False)]) @pytest.mark.parametrize("no_replace", [(True), (False)])
def test_special_phrases_csv_command(temp_db, mock_func_factory, tokenizer_mock, no_replace): def test_special_phrases_csv_command(self, src_dir, mock_func_factory, no_replace):
func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases') func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
testdata = SRC_DIR / 'test' / 'testdb' testdata = src_dir / 'test' / 'testdb'
csv_path = str((testdata / 'full_en_phrases_test.csv').resolve()) csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
if no_replace: if no_replace:
call_nominatim('special-phrases', '--import-from-csv', csv_path, '--no-replace') self.call_nominatim('special-phrases', '--import-from-csv', csv_path, '--no-replace')
else: else:
call_nominatim('special-phrases', '--import-from-csv', csv_path) self.call_nominatim('special-phrases', '--import-from-csv', csv_path)
assert func.called == 1 assert func.called == 1
@pytest.mark.parametrize("command,func", [ @pytest.mark.parametrize("command,func", [
('word-counts', 'recompute_word_counts'), ('word-counts', 'recompute_word_counts'),
('address-levels', 'load_address_levels_from_file'), ('address-levels', 'load_address_levels_from_file'),
('wiki-data', 'import_wikipedia_articles'), ('wiki-data', 'import_wikipedia_articles'),
('importance', 'recompute_importance'), ('importance', 'recompute_importance'),
('website', 'setup_website'), ('website', 'setup_website'),
]) ])
def test_refresh_command(mock_func_factory, temp_db, command, func, tokenizer_mock): def test_refresh_command(self, mock_func_factory, command, func):
func_mock = mock_func_factory(nominatim.tools.refresh, func) func_mock = mock_func_factory(nominatim.tools.refresh, func)
assert 0 == call_nominatim('refresh', '--' + command) assert self.call_nominatim('refresh', '--' + command) == 0
assert func_mock.called == 1 assert func_mock.called == 1
def test_refresh_postcodes(mock_func_factory, temp_db, tokenizer_mock): def test_refresh_postcodes(self, mock_func_factory):
func_mock = mock_func_factory(nominatim.tools.postcodes, 'update_postcodes') func_mock = mock_func_factory(nominatim.tools.postcodes, 'update_postcodes')
idx_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_postcodes') idx_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_postcodes')
assert 0 == call_nominatim('refresh', '--postcodes') assert self.call_nominatim('refresh', '--postcodes') == 0
assert func_mock.called == 1 assert func_mock.called == 1
assert idx_mock.called == 1
def test_refresh_create_functions(mock_func_factory, temp_db, tokenizer_mock): def test_refresh_create_functions(self, mock_func_factory):
func_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions') func_mock = mock_func_factory(nominatim.tools.refresh, 'create_functions')
assert 0 == call_nominatim('refresh', '--functions') assert self.call_nominatim('refresh', '--functions') == 0
assert func_mock.called == 1 assert func_mock.called == 1
assert tokenizer_mock.update_sql_functions_called assert self.tokenizer_mock.update_sql_functions_called
def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db, tokenizer_mock): def test_refresh_importance_computed_after_wiki_import(self, monkeypatch):
calls = [] calls = []
monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles', monkeypatch.setattr(nominatim.tools.refresh, 'import_wikipedia_articles',
lambda *args, **kwargs: calls.append('import') or 0) lambda *args, **kwargs: calls.append('import') or 0)
monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance', monkeypatch.setattr(nominatim.tools.refresh, 'recompute_importance',
lambda *args, **kwargs: calls.append('update')) lambda *args, **kwargs: calls.append('update'))
assert 0 == call_nominatim('refresh', '--importance', '--wiki-data') assert self.call_nominatim('refresh', '--importance', '--wiki-data') == 0
assert calls == ['import', 'update'] assert calls == ['import', 'update']
def test_serve_command(mock_func_factory):
func = mock_func_factory(nominatim.cli, 'run_php_server')
call_nominatim('serve')
assert func.called == 1
@pytest.mark.parametrize("params", [
('search', '--query', 'new'),
('reverse', '--lat', '0', '--lon', '0'),
('lookup', '--id', 'N1'),
('details', '--node', '1'),
('details', '--way', '1'),
('details', '--relation', '1'),
('details', '--place_id', '10001'),
('status',)
])
def test_api_commands_simple(mock_func_factory, params):
mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
assert 0 == call_nominatim(*params)
assert mock_run_api.called == 1
assert mock_run_api.last_args[0] == params[0]

81
test/python/test_cli_replication.py
View File

@ -3,7 +3,6 @@ Tests for replication command of command-line interface wrapper.
""" """
import datetime as dt import datetime as dt
import time import time
from pathlib import Path
import pytest import pytest
@ -14,18 +13,6 @@ from nominatim.db import status
from mocks import MockParamCapture from mocks import MockParamCapture
SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()
def call_nominatim(*args):
return nominatim.cli.nominatim(module_dir='build/module',
osm2pgsql_path='build/osm2pgsql/osm2pgsql',
phplib_dir=str(SRC_DIR / 'lib-php'),
data_dir=str(SRC_DIR / 'data'),
phpcgi_path='/usr/bin/php-cgi',
sqllib_dir=str(SRC_DIR / 'lib-sql'),
config_dir=str(SRC_DIR / 'settings'),
cli_args=['replication'] + list(args))
@pytest.fixture @pytest.fixture
def tokenizer_mock(monkeypatch): def tokenizer_mock(monkeypatch):
class DummyTokenizer: class DummyTokenizer:
@ -40,23 +27,14 @@ def tokenizer_mock(monkeypatch):
self.finalize_import_called = True self.finalize_import_called = True
tok = DummyTokenizer() tok = DummyTokenizer()
monkeypatch.setattr(nominatim.tokenizer.factory, 'get_tokenizer_for_db' , monkeypatch.setattr(nominatim.tokenizer.factory, 'get_tokenizer_for_db',
lambda *args: tok) lambda *args: tok)
monkeypatch.setattr(nominatim.tokenizer.factory, 'create_tokenizer' , monkeypatch.setattr(nominatim.tokenizer.factory, 'create_tokenizer',
lambda *args: tok) lambda *args: tok)
return tok return tok
@pytest.fixture
def index_mock(monkeypatch, tokenizer_mock):
mock = MockParamCapture()
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', mock)
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', mock)
return mock
@pytest.fixture @pytest.fixture
def mock_func_factory(monkeypatch): def mock_func_factory(monkeypatch):
def get_mock(module, func): def get_mock(module, func):
@ -70,69 +48,84 @@ def mock_func_factory(monkeypatch):
@pytest.fixture @pytest.fixture
def init_status(temp_db_conn, status_table): def init_status(temp_db_conn, status_table):
status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1) status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1)
return 1
@pytest.fixture
def index_mock(monkeypatch, tokenizer_mock, init_status):
mock = MockParamCapture()
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', mock)
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', mock)
return mock
@pytest.fixture @pytest.fixture
def update_mock(mock_func_factory, init_status, tokenizer_mock): def update_mock(mock_func_factory, init_status, tokenizer_mock):
return mock_func_factory(nominatim.tools.replication, 'update') return mock_func_factory(nominatim.tools.replication, 'update')
@pytest.mark.parametrize("params,func", [
class TestCliReplication:
@pytest.fixture(autouse=True)
def setup_cli_call(self, cli_call, temp_db):
self.call_nominatim = lambda *args: cli_call('replication', *args)
@pytest.mark.parametrize("params,func", [
(('--init', '--no-update-functions'), 'init_replication'), (('--init', '--no-update-functions'), 'init_replication'),
(('--check-for-updates',), 'check_for_updates') (('--check-for-updates',), 'check_for_updates')
]) ])
def test_replication_command(mock_func_factory, temp_db, params, func): def test_replication_command(self, mock_func_factory, params, func):
func_mock = mock_func_factory(nominatim.tools.replication, func) func_mock = mock_func_factory(nominatim.tools.replication, func)
assert 0 == call_nominatim(*params) assert self.call_nominatim(*params) == 0
assert func_mock.called == 1 assert func_mock.called == 1
def test_replication_update_bad_interval(monkeypatch, temp_db): def test_replication_update_bad_interval(self, monkeypatch):
monkeypatch.setenv('NOMINATIM_REPLICATION_UPDATE_INTERVAL', 'xx') monkeypatch.setenv('NOMINATIM_REPLICATION_UPDATE_INTERVAL', 'xx')
assert call_nominatim() == 1 assert self.call_nominatim() == 1
def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db): def test_replication_update_bad_interval_for_geofabrik(self, monkeypatch):
monkeypatch.setenv('NOMINATIM_REPLICATION_URL', monkeypatch.setenv('NOMINATIM_REPLICATION_URL',
'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates') 'https://download.geofabrik.de/europe/italy-updates')
assert call_nominatim() == 1 assert self.call_nominatim() == 1
def test_replication_update_once_no_index(update_mock): def test_replication_update_once_no_index(self, update_mock):
assert 0 == call_nominatim('--once', '--no-index') assert self.call_nominatim('--once', '--no-index') == 0
assert str(update_mock.last_args[1]['osm2pgsql']) == 'build/osm2pgsql/osm2pgsql' assert str(update_mock.last_args[1]['osm2pgsql']) == 'OSM2PGSQL NOT AVAILABLE'
def test_replication_update_custom_osm2pgsql(monkeypatch, update_mock): def test_replication_update_custom_osm2pgsql(self, monkeypatch, update_mock):
monkeypatch.setenv('NOMINATIM_OSM2PGSQL_BINARY', '/secret/osm2pgsql') monkeypatch.setenv('NOMINATIM_OSM2PGSQL_BINARY', '/secret/osm2pgsql')
assert 0 == call_nominatim('--once', '--no-index') assert self.call_nominatim('--once', '--no-index') == 0
assert str(update_mock.last_args[1]['osm2pgsql']) == '/secret/osm2pgsql' assert str(update_mock.last_args[1]['osm2pgsql']) == '/secret/osm2pgsql'
def test_replication_update_custom_threads(update_mock): def test_replication_update_custom_threads(self, update_mock):
assert 0 == call_nominatim('--once', '--no-index', '--threads', '4') assert self.call_nominatim('--once', '--no-index', '--threads', '4') == 0
assert update_mock.last_args[1]['threads'] == 4 assert update_mock.last_args[1]['threads'] == 4
def test_replication_update_continuous(monkeypatch, init_status, index_mock): def test_replication_update_continuous(self, monkeypatch, index_mock):
states = [nominatim.tools.replication.UpdateState.UP_TO_DATE, states = [nominatim.tools.replication.UpdateState.UP_TO_DATE,
nominatim.tools.replication.UpdateState.UP_TO_DATE] nominatim.tools.replication.UpdateState.UP_TO_DATE]
monkeypatch.setattr(nominatim.tools.replication, 'update', monkeypatch.setattr(nominatim.tools.replication, 'update',
lambda *args, **kwargs: states.pop()) lambda *args, **kwargs: states.pop())
with pytest.raises(IndexError): with pytest.raises(IndexError):
call_nominatim() self.call_nominatim()
assert index_mock.called == 4 assert index_mock.called == 4
def test_replication_update_continuous_no_change(monkeypatch, init_status, index_mock): def test_replication_update_continuous_no_change(self, monkeypatch, index_mock):
states = [nominatim.tools.replication.UpdateState.NO_CHANGES, states = [nominatim.tools.replication.UpdateState.NO_CHANGES,
nominatim.tools.replication.UpdateState.UP_TO_DATE] nominatim.tools.replication.UpdateState.UP_TO_DATE]
monkeypatch.setattr(nominatim.tools.replication, 'update', monkeypatch.setattr(nominatim.tools.replication, 'update',
@ -142,7 +135,7 @@ def test_replication_update_continuous_no_change(monkeypatch, init_status, index
monkeypatch.setattr(time, 'sleep', sleep_mock) monkeypatch.setattr(time, 'sleep', sleep_mock)
with pytest.raises(IndexError): with pytest.raises(IndexError):
call_nominatim() self.call_nominatim()
assert index_mock.called == 2 assert index_mock.called == 2
assert sleep_mock.called == 1 assert sleep_mock.called == 1

80
test/python/test_config.py
View File

@ -1,66 +1,72 @@
""" """
Test for loading dotenv configuration. Test for loading dotenv configuration.
""" """
from pathlib import Path
import pytest import pytest
from nominatim.config import Configuration from nominatim.config import Configuration
from nominatim.errors import UsageError from nominatim.errors import UsageError
DEFCFG_DIR = Path(__file__) / '..' / '..' / '..' / 'settings' @pytest.fixture
def make_config(src_dir):
""" Create a configuration object from the given project directory.
"""
def _mk_config(project_dir=None):
return Configuration(project_dir, src_dir / 'settings')
def test_no_project_dir(): return _mk_config
config = Configuration(None, DEFCFG_DIR)
def test_no_project_dir(make_config):
config = make_config()
assert config.DATABASE_WEBUSER == 'www-data' assert config.DATABASE_WEBUSER == 'www-data'
@pytest.mark.parametrize("val", ('apache', '"apache"')) @pytest.mark.parametrize("val", ('apache', '"apache"'))
def test_prefer_project_setting_over_default(val, tmp_path): def test_prefer_project_setting_over_default(make_config, val, tmp_path):
envfile = tmp_path / '.env' envfile = tmp_path / '.env'
envfile.write_text('NOMINATIM_DATABASE_WEBUSER={}\n'.format(val)) envfile.write_text('NOMINATIM_DATABASE_WEBUSER={}\n'.format(val))
config = Configuration(Path(tmp_path), DEFCFG_DIR) config = make_config(tmp_path)
assert config.DATABASE_WEBUSER == 'apache' assert config.DATABASE_WEBUSER == 'apache'
def test_prefer_os_environ_over_project_setting(monkeypatch, tmp_path): def test_prefer_os_environ_over_project_setting(make_config, monkeypatch, tmp_path):
envfile = tmp_path / '.env' envfile = tmp_path / '.env'
envfile.write_text('NOMINATIM_DATABASE_WEBUSER=apache\n') envfile.write_text('NOMINATIM_DATABASE_WEBUSER=apache\n')
monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'nobody') monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'nobody')
config = Configuration(Path(tmp_path), DEFCFG_DIR) config = make_config(tmp_path)
assert config.DATABASE_WEBUSER == 'nobody' assert config.DATABASE_WEBUSER == 'nobody'
def test_get_os_env_add_defaults(monkeypatch): def test_get_os_env_add_defaults(make_config, monkeypatch):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.delenv('NOMINATIM_DATABASE_WEBUSER', raising=False) monkeypatch.delenv('NOMINATIM_DATABASE_WEBUSER', raising=False)
assert config.get_os_env()['NOMINATIM_DATABASE_WEBUSER'] == 'www-data' assert config.get_os_env()['NOMINATIM_DATABASE_WEBUSER'] == 'www-data'
def test_get_os_env_prefer_os_environ(monkeypatch): def test_get_os_env_prefer_os_environ(make_config, monkeypatch):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'nobody') monkeypatch.setenv('NOMINATIM_DATABASE_WEBUSER', 'nobody')
assert config.get_os_env()['NOMINATIM_DATABASE_WEBUSER'] == 'nobody' assert config.get_os_env()['NOMINATIM_DATABASE_WEBUSER'] == 'nobody'
def test_get_libpq_dsn_convert_default(): def test_get_libpq_dsn_convert_default(make_config):
config = Configuration(None, DEFCFG_DIR) config = make_config()
assert config.get_libpq_dsn() == 'dbname=nominatim' assert config.get_libpq_dsn() == 'dbname=nominatim'
def test_get_libpq_dsn_convert_php(monkeypatch): def test_get_libpq_dsn_convert_php(make_config, monkeypatch):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_DATABASE_DSN', monkeypatch.setenv('NOMINATIM_DATABASE_DSN',
'pgsql:dbname=gis;password=foo;host=localhost') 'pgsql:dbname=gis;password=foo;host=localhost')
@ -71,8 +77,8 @@ def test_get_libpq_dsn_convert_php(monkeypatch):
@pytest.mark.parametrize("val,expect", [('foo bar', "'foo bar'"), @pytest.mark.parametrize("val,expect", [('foo bar', "'foo bar'"),
("xy'z", "xy\\'z"), ("xy'z", "xy\\'z"),
]) ])
def test_get_libpq_dsn_convert_php_special_chars(monkeypatch, val, expect): def test_get_libpq_dsn_convert_php_special_chars(make_config, monkeypatch, val, expect):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_DATABASE_DSN', monkeypatch.setenv('NOMINATIM_DATABASE_DSN',
'pgsql:dbname=gis;password={}'.format(val)) 'pgsql:dbname=gis;password={}'.format(val))
@ -80,8 +86,8 @@ def test_get_libpq_dsn_convert_php_special_chars(monkeypatch, val, expect):
assert config.get_libpq_dsn() == "dbname=gis password={}".format(expect) assert config.get_libpq_dsn() == "dbname=gis password={}".format(expect)
def test_get_libpq_dsn_convert_libpq(monkeypatch): def test_get_libpq_dsn_convert_libpq(make_config, monkeypatch):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_DATABASE_DSN', monkeypatch.setenv('NOMINATIM_DATABASE_DSN',
'host=localhost dbname=gis password=foo') 'host=localhost dbname=gis password=foo')
@ -92,24 +98,24 @@ def test_get_libpq_dsn_convert_libpq(monkeypatch):
@pytest.mark.parametrize("value,result", @pytest.mark.parametrize("value,result",
[(x, True) for x in ('1', 'true', 'True', 'yes', 'YES')] + [(x, True) for x in ('1', 'true', 'True', 'yes', 'YES')] +
[(x, False) for x in ('0', 'false', 'no', 'NO', 'x')]) [(x, False) for x in ('0', 'false', 'no', 'NO', 'x')])
def test_get_bool(monkeypatch, value, result): def test_get_bool(make_config, monkeypatch, value, result):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_FOOBAR', value) monkeypatch.setenv('NOMINATIM_FOOBAR', value)
assert config.get_bool('FOOBAR') == result assert config.get_bool('FOOBAR') == result
def test_get_bool_empty(): def test_get_bool_empty(make_config):
config = Configuration(None, DEFCFG_DIR) config = make_config()
assert config.DATABASE_MODULE_PATH == '' assert config.DATABASE_MODULE_PATH == ''
assert config.get_bool('DATABASE_MODULE_PATH') == False assert not config.get_bool('DATABASE_MODULE_PATH')
@pytest.mark.parametrize("value,result", [('0', 0), ('1', 1), @pytest.mark.parametrize("value,result", [('0', 0), ('1', 1),
('85762513444', 85762513444)]) ('85762513444', 85762513444)])
def test_get_int_success(monkeypatch, value, result): def test_get_int_success(make_config, monkeypatch, value, result):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_FOOBAR', value) monkeypatch.setenv('NOMINATIM_FOOBAR', value)
@ -117,8 +123,8 @@ def test_get_int_success(monkeypatch, value, result):
@pytest.mark.parametrize("value", ['1b', 'fg', '0x23']) @pytest.mark.parametrize("value", ['1b', 'fg', '0x23'])
def test_get_int_bad_values(monkeypatch, value): def test_get_int_bad_values(make_config, monkeypatch, value):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_FOOBAR', value) monkeypatch.setenv('NOMINATIM_FOOBAR', value)
@ -126,8 +132,8 @@ def test_get_int_bad_values(monkeypatch, value):
config.get_int('FOOBAR') config.get_int('FOOBAR')
def test_get_int_empty(): def test_get_int_empty(make_config):
config = Configuration(None, DEFCFG_DIR) config = make_config()
assert config.DATABASE_MODULE_PATH == '' assert config.DATABASE_MODULE_PATH == ''
@ -135,19 +141,19 @@ def test_get_int_empty():
config.get_int('DATABASE_MODULE_PATH') config.get_int('DATABASE_MODULE_PATH')
def test_get_import_style_intern(monkeypatch): def test_get_import_style_intern(make_config, src_dir, monkeypatch):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', 'street') monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', 'street')
expected = DEFCFG_DIR / 'import-street.style' expected = src_dir / 'settings' / 'import-street.style'
assert config.get_import_style_file() == expected assert config.get_import_style_file() == expected
@pytest.mark.parametrize("value", ['custom', '/foo/bar.stye']) @pytest.mark.parametrize("value", ['custom', '/foo/bar.stye'])
def test_get_import_style_intern(monkeypatch, value): def test_get_import_style_extern(make_config, monkeypatch, value):
config = Configuration(None, DEFCFG_DIR) config = make_config()
monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', value) monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', value)

7
test/python/test_db_async_connection.py
View File

@ -6,15 +6,14 @@ import concurrent.futures
import pytest import pytest
import psycopg2 import psycopg2
from psycopg2.extras import wait_select
from nominatim.db.async_connection import DBConnection, DeadlockHandler from nominatim.db.async_connection import DBConnection, DeadlockHandler
@pytest.fixture @pytest.fixture
def conn(temp_db): def conn(temp_db):
with closing(DBConnection('dbname=' + temp_db)) as c: with closing(DBConnection('dbname=' + temp_db)) as connection:
yield c yield connection
@pytest.fixture @pytest.fixture
@ -106,5 +105,3 @@ def test_deadlock(simple_conns):
future.result() future.result()
assert len(deadlock_check) == 1 assert len(deadlock_check) == 1

24
test/python/test_db_connection.py
View File

@ -7,28 +7,28 @@ import psycopg2
from nominatim.db.connection import connect, get_pg_env from nominatim.db.connection import connect, get_pg_env
@pytest.fixture @pytest.fixture
def db(temp_db): def db(dsn):
with connect('dbname=' + temp_db) as conn: with connect(dsn) as conn:
yield conn yield conn
def test_connection_table_exists(db, table_factory): def test_connection_table_exists(db, table_factory):
assert db.table_exists('foobar') == False assert not db.table_exists('foobar')
table_factory('foobar') table_factory('foobar')
assert db.table_exists('foobar') == True assert db.table_exists('foobar')
def test_connection_index_exists(db, temp_db_cursor): def test_connection_index_exists(db, table_factory, temp_db_cursor):
assert db.index_exists('some_index') == False assert not db.index_exists('some_index')
temp_db_cursor.execute('CREATE TABLE foobar (id INT)') table_factory('foobar')
temp_db_cursor.execute('CREATE INDEX some_index ON foobar(id)') temp_db_cursor.execute('CREATE INDEX some_index ON foobar(id)')
assert db.index_exists('some_index') == True assert db.index_exists('some_index')
assert db.index_exists('some_index', table='foobar') == True assert db.index_exists('some_index', table='foobar')
assert db.index_exists('some_index', table='bar') == False assert not db.index_exists('some_index', table='bar')
def test_drop_table_existing(db, table_factory): def test_drop_table_existing(db, table_factory):
@ -55,9 +55,7 @@ def test_connection_server_version_tuple(db):
assert ver[0] > 8 assert ver[0] > 8
def test_connection_postgis_version_tuple(db, temp_db_cursor): def test_connection_postgis_version_tuple(db, temp_db_with_extensions):
temp_db_cursor.execute('CREATE EXTENSION postgis')
ver = db.postgis_version_tuple() ver = db.postgis_version_tuple()
assert isinstance(ver, tuple) assert isinstance(ver, tuple)

22
test/python/test_db_properties.py
View File

@ -6,26 +6,32 @@ import pytest
from nominatim.db import properties from nominatim.db import properties
@pytest.fixture @pytest.fixture
def prop_table(table_factory): def property_factory(property_table, temp_db_cursor):
table_factory('nominatim_properties', 'property TEXT, value TEXT') """ A function fixture that adds a property into the property table.
"""
def _add_property(name, value):
temp_db_cursor.execute("INSERT INTO nominatim_properties VALUES(%s, %s)",
(name, value))
return _add_property
def test_get_property_existing(prop_table, temp_db_conn, temp_db_cursor): def test_get_property_existing(property_factory, temp_db_conn):
temp_db_cursor.execute("INSERT INTO nominatim_properties VALUES('foo', 'bar')") property_factory('foo', 'bar')
assert properties.get_property(temp_db_conn, 'foo') == 'bar' assert properties.get_property(temp_db_conn, 'foo') == 'bar'
def test_get_property_unknown(prop_table, temp_db_conn, temp_db_cursor): def test_get_property_unknown(property_factory, temp_db_conn):
temp_db_cursor.execute("INSERT INTO nominatim_properties VALUES('other', 'bar')") property_factory('other', 'bar')
assert properties.get_property(temp_db_conn, 'foo') is None assert properties.get_property(temp_db_conn, 'foo') is None
@pytest.mark.parametrize("prefill", (True, False)) @pytest.mark.parametrize("prefill", (True, False))
def test_set_property_new(prop_table, temp_db_conn, temp_db_cursor, prefill): def test_set_property_new(property_factory, temp_db_conn, temp_db_cursor, prefill):
if prefill: if prefill:
temp_db_cursor.execute("INSERT INTO nominatim_properties VALUES('something', 'bar')") property_factory('something', 'bar')
properties.set_property(temp_db_conn, 'something', 'else') properties.set_property(temp_db_conn, 'something', 'else')

2
test/python/test_db_sql_preprocessor.py
View File

@ -1,8 +1,6 @@
""" """
Tests for SQL preprocessing. Tests for SQL preprocessing.
""" """
from pathlib import Path
import pytest import pytest
@pytest.fixture @pytest.fixture

69
test/python/test_db_status.py
View File

@ -8,10 +8,6 @@ import pytest
import nominatim.db.status import nominatim.db.status
from nominatim.errors import UsageError from nominatim.errors import UsageError
def test_compute_database_date_place_empty(status_table, place_table, temp_db_conn):
with pytest.raises(UsageError):
nominatim.db.status.compute_database_date(temp_db_conn)
OSM_NODE_DATA = """\ OSM_NODE_DATA = """\
<osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/"> <osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
<node id="45673" visible="true" version="1" changeset="2047" timestamp="2006-01-27T22:09:10Z" user="Foo" uid="111" lat="48.7586670" lon="8.1343060"> <node id="45673" visible="true" version="1" changeset="2047" timestamp="2006-01-27T22:09:10Z" user="Foo" uid="111" lat="48.7586670" lon="8.1343060">
@ -24,7 +20,17 @@ def iso_date(date):
.replace(tzinfo=dt.timezone.utc) .replace(tzinfo=dt.timezone.utc)
def test_compute_database_date_valid(monkeypatch, status_table, place_row, temp_db_conn): @pytest.fixture(autouse=True)
def setup_status_table(status_table):
pass
def test_compute_database_date_place_empty(place_table, temp_db_conn):
with pytest.raises(UsageError):
nominatim.db.status.compute_database_date(temp_db_conn)
def test_compute_database_date_valid(monkeypatch, place_row, temp_db_conn):
place_row(osm_type='N', osm_id=45673) place_row(osm_type='N', osm_id=45673)
requested_url = [] requested_url = []
@ -40,7 +46,7 @@ def test_compute_database_date_valid(monkeypatch, status_table, place_row, temp_
assert date == iso_date('2006-01-27T22:09:10') assert date == iso_date('2006-01-27T22:09:10')
def test_compute_database_broken_api(monkeypatch, status_table, place_row, temp_db_conn): def test_compute_database_broken_api(monkeypatch, place_row, temp_db_conn):
place_row(osm_type='N', osm_id=45673) place_row(osm_type='N', osm_id=45673)
requested_url = [] requested_url = []
@ -51,53 +57,47 @@ def test_compute_database_broken_api(monkeypatch, status_table, place_row, temp_
monkeypatch.setattr(nominatim.db.status, "get_url", mock_url) monkeypatch.setattr(nominatim.db.status, "get_url", mock_url)
with pytest.raises(UsageError): with pytest.raises(UsageError):
date = nominatim.db.status.compute_database_date(temp_db_conn) nominatim.db.status.compute_database_date(temp_db_conn)
def test_set_status_empty_table(status_table, temp_db_conn, temp_db_cursor): def test_set_status_empty_table(temp_db_conn, temp_db_cursor):
date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc) date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
nominatim.db.status.set_status(temp_db_conn, date=date) nominatim.db.status.set_status(temp_db_conn, date=date)
temp_db_cursor.execute("SELECT * FROM import_status") assert temp_db_cursor.row_set("SELECT * FROM import_status") == \
{(date, None, True)}
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone() == [date, None, True]
def test_set_status_filled_table(status_table, temp_db_conn, temp_db_cursor): def test_set_status_filled_table(temp_db_conn, temp_db_cursor):
date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc) date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
nominatim.db.status.set_status(temp_db_conn, date=date) nominatim.db.status.set_status(temp_db_conn, date=date)
assert 1 == temp_db_cursor.scalar("SELECT count(*) FROM import_status") assert temp_db_cursor.table_rows('import_status') == 1
date = dt.datetime.fromordinal(1000100).replace(tzinfo=dt.timezone.utc) date = dt.datetime.fromordinal(1000100).replace(tzinfo=dt.timezone.utc)
nominatim.db.status.set_status(temp_db_conn, date=date, seq=456, indexed=False) nominatim.db.status.set_status(temp_db_conn, date=date, seq=456, indexed=False)
temp_db_cursor.execute("SELECT * FROM import_status") assert temp_db_cursor.row_set("SELECT * FROM import_status") == \
{(date, 456, False)}
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone() == [date, 456, False]
def test_set_status_missing_date(status_table, temp_db_conn, temp_db_cursor): def test_set_status_missing_date(temp_db_conn, temp_db_cursor):
date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc) date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
nominatim.db.status.set_status(temp_db_conn, date=date) nominatim.db.status.set_status(temp_db_conn, date=date)
assert 1 == temp_db_cursor.scalar("SELECT count(*) FROM import_status") assert temp_db_cursor.table_rows('import_status') == 1
nominatim.db.status.set_status(temp_db_conn, date=None, seq=456, indexed=False) nominatim.db.status.set_status(temp_db_conn, date=None, seq=456, indexed=False)
temp_db_cursor.execute("SELECT * FROM import_status") assert temp_db_cursor.row_set("SELECT * FROM import_status") == \
{(date, 456, False)}
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone() == [date, 456, False]
def test_get_status_empty_table(status_table, temp_db_conn): def test_get_status_empty_table(temp_db_conn):
assert nominatim.db.status.get_status(temp_db_conn) == (None, None, None) assert nominatim.db.status.get_status(temp_db_conn) == (None, None, None)
def test_get_status_success(status_table, temp_db_conn): def test_get_status_success(temp_db_conn):
date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc) date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
nominatim.db.status.set_status(temp_db_conn, date=date, seq=667, indexed=False) nominatim.db.status.set_status(temp_db_conn, date=date, seq=667, indexed=False)
@ -107,7 +107,7 @@ def test_get_status_success(status_table, temp_db_conn):
@pytest.mark.parametrize("old_state", [True, False]) @pytest.mark.parametrize("old_state", [True, False])
@pytest.mark.parametrize("new_state", [True, False]) @pytest.mark.parametrize("new_state", [True, False])
def test_set_indexed(status_table, temp_db_conn, temp_db_cursor, old_state, new_state): def test_set_indexed(temp_db_conn, temp_db_cursor, old_state, new_state):
date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc) date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
nominatim.db.status.set_status(temp_db_conn, date=date, indexed=old_state) nominatim.db.status.set_status(temp_db_conn, date=date, indexed=old_state)
nominatim.db.status.set_indexed(temp_db_conn, new_state) nominatim.db.status.set_indexed(temp_db_conn, new_state)
@ -115,18 +115,21 @@ def test_set_indexed(status_table, temp_db_conn, temp_db_cursor, old_state, new_
assert temp_db_cursor.scalar("SELECT indexed FROM import_status") == new_state assert temp_db_cursor.scalar("SELECT indexed FROM import_status") == new_state
def test_set_indexed_empty_status(status_table, temp_db_conn, temp_db_cursor): def test_set_indexed_empty_status(temp_db_conn, temp_db_cursor):
nominatim.db.status.set_indexed(temp_db_conn, True) nominatim.db.status.set_indexed(temp_db_conn, True)
assert temp_db_cursor.scalar("SELECT count(*) FROM import_status") == 0 assert temp_db_cursor.table_rows("import_status") == 0
def text_log_status(status_table, temp_db_conn): def test_log_status(temp_db_conn, temp_db_cursor):
date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc) date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
start = dt.datetime.now() - dt.timedelta(hours=1) start = dt.datetime.now() - dt.timedelta(hours=1)
nominatim.db.status.set_status(temp_db_conn, date=date, seq=56) nominatim.db.status.set_status(temp_db_conn, date=date, seq=56)
nominatim.db.status.log_status(temp_db_conn, start, 'index') nominatim.db.status.log_status(temp_db_conn, start, 'index')
assert temp_db_cursor.scalar("SELECT count(*) FROM import_osmosis_log") == 1 temp_db_conn.commit()
assert temp_db_cursor.scalar("SELECT seq FROM import_osmosis_log") == 56
assert temp_db_cursor.scalar("SELECT date FROM import_osmosis_log") == date assert temp_db_cursor.table_rows("import_osmosis_log") == 1
assert temp_db_cursor.scalar("SELECT batchseq FROM import_osmosis_log") == 56
assert temp_db_cursor.scalar("SELECT event FROM import_osmosis_log") == 'index'

16
test/python/test_db_utils.py
View File

@ -1,7 +1,6 @@
""" """
Tests for DB utility functions in db.utils Tests for DB utility functions in db.utils
""" """
import psycopg2
import pytest import pytest
import nominatim.db.utils as db_utils import nominatim.db.utils as db_utils
@ -13,10 +12,7 @@ def test_execute_file_success(dsn, temp_db_cursor, tmp_path):
db_utils.execute_file(dsn, tmpfile) db_utils.execute_file(dsn, tmpfile)
temp_db_cursor.execute('SELECT * FROM test') assert temp_db_cursor.row_set('SELECT * FROM test') == {(56, )}
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone()[0] == 56
def test_execute_file_bad_file(dsn, tmp_path): def test_execute_file_bad_file(dsn, tmp_path):
with pytest.raises(FileNotFoundError): with pytest.raises(FileNotFoundError):
@ -44,10 +40,7 @@ def test_execute_file_with_pre_code(dsn, tmp_path, temp_db_cursor):
db_utils.execute_file(dsn, tmpfile, pre_code='CREATE TABLE test (id INT)') db_utils.execute_file(dsn, tmpfile, pre_code='CREATE TABLE test (id INT)')
temp_db_cursor.execute('SELECT * FROM test') assert temp_db_cursor.row_set('SELECT * FROM test') == {(4, )}
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone()[0] == 4
def test_execute_file_with_post_code(dsn, tmp_path, temp_db_cursor): def test_execute_file_with_post_code(dsn, tmp_path, temp_db_cursor):
@ -56,7 +49,4 @@ def test_execute_file_with_post_code(dsn, tmp_path, temp_db_cursor):
db_utils.execute_file(dsn, tmpfile, post_code='INSERT INTO test VALUES(23)') db_utils.execute_file(dsn, tmpfile, post_code='INSERT INTO test VALUES(23)')
temp_db_cursor.execute('SELECT * FROM test') assert temp_db_cursor.row_set('SELECT * FROM test') == {(23, )}
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone()[0] == 23

93
test/python/test_indexing.py
View File

@ -2,7 +2,6 @@
Tests for running the indexing. Tests for running the indexing.
""" """
import itertools import itertools
import psycopg2
import pytest import pytest
from nominatim.indexer import indexer from nominatim.indexer import indexer
@ -64,7 +63,8 @@ class IndexerTestDB:
END; END;
$$ LANGUAGE plpgsql STABLE; $$ LANGUAGE plpgsql STABLE;
""") """)
cur.execute("""CREATE OR REPLACE FUNCTION get_interpolation_address(in_address HSTORE, wayid BIGINT) cur.execute("""CREATE OR REPLACE FUNCTION
get_interpolation_address(in_address HSTORE, wayid BIGINT)
RETURNS HSTORE AS $$ RETURNS HSTORE AS $$
BEGIN BEGIN
RETURN in_address; RETURN in_address;
@ -120,7 +120,8 @@ class IndexerTestDB:
return self.scalar('SELECT count(*) from placex where indexed_status > 0') return self.scalar('SELECT count(*) from placex where indexed_status > 0')
def osmline_unindexed(self): def osmline_unindexed(self):
return self.scalar('SELECT count(*) from location_property_osmline where indexed_status > 0') return self.scalar("""SELECT count(*) from location_property_osmline
WHERE indexed_status > 0""")
@pytest.fixture @pytest.fixture
@ -140,37 +141,41 @@ def test_index_all_by_rank(test_db, threads, test_tokenizer):
test_db.add_place(rank_address=rank, rank_search=rank) test_db.add_place(rank_address=rank, rank_search=rank)
test_db.add_osmline() test_db.add_osmline()
assert 31 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 31
assert 1 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 1
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
idx.index_by_rank(0, 30) idx.index_by_rank(0, 30)
assert 0 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 0
assert 0 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 0
assert 0 == test_db.scalar("""SELECT count(*) from placex assert test_db.scalar("""SELECT count(*) from placex
WHERE indexed_status = 0 and indexed_date is null""") WHERE indexed_status = 0 and indexed_date is null""") == 0
# ranks come in order of rank address # ranks come in order of rank address
assert 0 == test_db.scalar(""" assert test_db.scalar("""
SELECT count(*) FROM placex p WHERE rank_address > 0 SELECT count(*) FROM placex p WHERE rank_address > 0
AND indexed_date >= (SELECT min(indexed_date) FROM placex o AND indexed_date >= (SELECT min(indexed_date) FROM placex o
WHERE p.rank_address < o.rank_address)""") WHERE p.rank_address < o.rank_address)""") == 0
# placex rank < 30 objects come before interpolations # placex rank < 30 objects come before interpolations
assert 0 == test_db.scalar( assert test_db.scalar(
"""SELECT count(*) FROM placex WHERE rank_address < 30 """SELECT count(*) FROM placex WHERE rank_address < 30
AND indexed_date > (SELECT min(indexed_date) FROM location_property_osmline)""") AND indexed_date >
(SELECT min(indexed_date) FROM location_property_osmline)""") == 0
# placex rank = 30 objects come after interpolations # placex rank = 30 objects come after interpolations
assert 0 == test_db.scalar( assert test_db.scalar(
"""SELECT count(*) FROM placex WHERE rank_address = 30 """SELECT count(*) FROM placex WHERE rank_address = 30
AND indexed_date < (SELECT max(indexed_date) FROM location_property_osmline)""") AND indexed_date <
(SELECT max(indexed_date) FROM location_property_osmline)""") == 0
# rank 0 comes after rank 29 and before rank 30 # rank 0 comes after rank 29 and before rank 30
assert 0 == test_db.scalar( assert test_db.scalar(
"""SELECT count(*) FROM placex WHERE rank_address < 30 """SELECT count(*) FROM placex WHERE rank_address < 30
AND indexed_date > (SELECT min(indexed_date) FROM placex WHERE rank_address = 0)""") AND indexed_date >
assert 0 == test_db.scalar( (SELECT min(indexed_date) FROM placex WHERE rank_address = 0)""") == 0
assert test_db.scalar(
"""SELECT count(*) FROM placex WHERE rank_address = 30 """SELECT count(*) FROM placex WHERE rank_address = 30
AND indexed_date < (SELECT max(indexed_date) FROM placex WHERE rank_address = 0)""") AND indexed_date <
(SELECT max(indexed_date) FROM placex WHERE rank_address = 0)""") == 0
@pytest.mark.parametrize("threads", [1, 15]) @pytest.mark.parametrize("threads", [1, 15])
@ -179,19 +184,19 @@ def test_index_partial_without_30(test_db, threads, test_tokenizer):
test_db.add_place(rank_address=rank, rank_search=rank) test_db.add_place(rank_address=rank, rank_search=rank)
test_db.add_osmline() test_db.add_osmline()
assert 31 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 31
assert 1 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 1
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', idx = indexer.Indexer('dbname=test_nominatim_python_unittest',
test_tokenizer, threads) test_tokenizer, threads)
idx.index_by_rank(4, 15) idx.index_by_rank(4, 15)
assert 19 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 19
assert 1 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 1
assert 0 == test_db.scalar(""" assert test_db.scalar("""
SELECT count(*) FROM placex SELECT count(*) FROM placex
WHERE indexed_status = 0 AND not rank_address between 4 and 15""") WHERE indexed_status = 0 AND not rank_address between 4 and 15""") == 0
@pytest.mark.parametrize("threads", [1, 15]) @pytest.mark.parametrize("threads", [1, 15])
@ -200,18 +205,18 @@ def test_index_partial_with_30(test_db, threads, test_tokenizer):
test_db.add_place(rank_address=rank, rank_search=rank) test_db.add_place(rank_address=rank, rank_search=rank)
test_db.add_osmline() test_db.add_osmline()
assert 31 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 31
assert 1 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 1
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
idx.index_by_rank(28, 30) idx.index_by_rank(28, 30)
assert 27 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 27
assert 0 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 0
assert 0 == test_db.scalar(""" assert test_db.scalar("""
SELECT count(*) FROM placex SELECT count(*) FROM placex
WHERE indexed_status = 0 AND rank_address between 1 and 27""") WHERE indexed_status = 0 AND rank_address between 1 and 27""") == 0
@pytest.mark.parametrize("threads", [1, 15]) @pytest.mark.parametrize("threads", [1, 15])
def test_index_boundaries(test_db, threads, test_tokenizer): def test_index_boundaries(test_db, threads, test_tokenizer):
@ -221,18 +226,18 @@ def test_index_boundaries(test_db, threads, test_tokenizer):
test_db.add_place(rank_address=rank, rank_search=rank) test_db.add_place(rank_address=rank, rank_search=rank)
test_db.add_osmline() test_db.add_osmline()
assert 37 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 37
assert 1 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 1
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
idx.index_boundaries(0, 30) idx.index_boundaries(0, 30)
assert 31 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 31
assert 1 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 1
assert 0 == test_db.scalar(""" assert test_db.scalar("""
SELECT count(*) FROM placex SELECT count(*) FROM placex
WHERE indexed_status = 0 AND class != 'boundary'""") WHERE indexed_status = 0 AND class != 'boundary'""") == 0
@pytest.mark.parametrize("threads", [1, 15]) @pytest.mark.parametrize("threads", [1, 15])
@ -245,8 +250,8 @@ def test_index_postcodes(test_db, threads, test_tokenizer):
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
idx.index_postcodes() idx.index_postcodes()
assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode assert test_db.scalar("""SELECT count(*) FROM location_postcode
WHERE indexed_status != 0""") WHERE indexed_status != 0""") == 0
@pytest.mark.parametrize("analyse", [True, False]) @pytest.mark.parametrize("analyse", [True, False])
@ -262,10 +267,10 @@ def test_index_full(test_db, analyse, test_tokenizer):
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, 4) idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, 4)
idx.index_full(analyse=analyse) idx.index_full(analyse=analyse)
assert 0 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 0
assert 0 == test_db.osmline_unindexed() assert test_db.osmline_unindexed() == 0
assert 0 == test_db.scalar("""SELECT count(*) FROM location_postcode assert test_db.scalar("""SELECT count(*) FROM location_postcode
WHERE indexed_status != 0""") WHERE indexed_status != 0""") == 0
@pytest.mark.parametrize("threads", [1, 15]) @pytest.mark.parametrize("threads", [1, 15])
@ -278,4 +283,4 @@ def test_index_reopen_connection(test_db, threads, monkeypatch, test_tokenizer):
idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads) idx = indexer.Indexer('dbname=test_nominatim_python_unittest', test_tokenizer, threads)
idx.index_by_rank(28, 30) idx.index_by_rank(28, 30)
assert 0 == test_db.placex_unindexed() assert test_db.placex_unindexed() == 0

24
test/python/test_tokenizer_factory.py
View File

@ -1,7 +1,6 @@
""" """
Tests for creating new tokenizers. Tests for creating new tokenizers.
""" """
import importlib
import pytest import pytest
from nominatim.db import properties from nominatim.db import properties
@ -10,13 +9,12 @@ from nominatim.errors import UsageError
from dummy_tokenizer import DummyTokenizer from dummy_tokenizer import DummyTokenizer
@pytest.fixture @pytest.fixture
def test_config(def_config, tmp_path): def test_config(def_config, tmp_path, property_table, tokenizer_mock):
def_config.project_dir = tmp_path def_config.project_dir = tmp_path
return def_config return def_config
def test_setup_dummy_tokenizer(temp_db_conn, test_config, def test_setup_dummy_tokenizer(temp_db_conn, test_config):
tokenizer_mock, property_table):
tokenizer = factory.create_tokenizer(test_config) tokenizer = factory.create_tokenizer(test_config)
assert isinstance(tokenizer, DummyTokenizer) assert isinstance(tokenizer, DummyTokenizer)
@ -26,7 +24,7 @@ def test_setup_dummy_tokenizer(temp_db_conn, test_config,
assert properties.get_property(temp_db_conn, 'tokenizer') == 'dummy' assert properties.get_property(temp_db_conn, 'tokenizer') == 'dummy'
def test_setup_tokenizer_dir_exists(test_config, tokenizer_mock, property_table): def test_setup_tokenizer_dir_exists(test_config):
(test_config.project_dir / 'tokenizer').mkdir() (test_config.project_dir / 'tokenizer').mkdir()
tokenizer = factory.create_tokenizer(test_config) tokenizer = factory.create_tokenizer(test_config)
@ -35,21 +33,22 @@ def test_setup_tokenizer_dir_exists(test_config, tokenizer_mock, property_table)
assert tokenizer.init_state == "new" assert tokenizer.init_state == "new"
def test_setup_tokenizer_dir_failure(test_config, tokenizer_mock, property_table): def test_setup_tokenizer_dir_failure(test_config):
(test_config.project_dir / 'tokenizer').write_text("foo") (test_config.project_dir / 'tokenizer').write_text("foo")
with pytest.raises(UsageError): with pytest.raises(UsageError):
factory.create_tokenizer(test_config) factory.create_tokenizer(test_config)
def test_setup_bad_tokenizer_name(test_config, monkeypatch): def test_setup_bad_tokenizer_name(def_config, tmp_path, monkeypatch):
def_config.project_dir = tmp_path
monkeypatch.setenv('NOMINATIM_TOKENIZER', 'dummy') monkeypatch.setenv('NOMINATIM_TOKENIZER', 'dummy')
with pytest.raises(UsageError): with pytest.raises(UsageError):
factory.create_tokenizer(test_config) factory.create_tokenizer(def_config)
def test_load_tokenizer(temp_db_conn, test_config,
tokenizer_mock, property_table): def test_load_tokenizer(test_config):
factory.create_tokenizer(test_config) factory.create_tokenizer(test_config)
tokenizer = factory.get_tokenizer_for_db(test_config) tokenizer = factory.get_tokenizer_for_db(test_config)
@ -58,7 +57,7 @@ def test_load_tokenizer(temp_db_conn, test_config,
assert tokenizer.init_state == "loaded" assert tokenizer.init_state == "loaded"
def test_load_no_tokenizer_dir(test_config, tokenizer_mock, property_table): def test_load_no_tokenizer_dir(test_config):
factory.create_tokenizer(test_config) factory.create_tokenizer(test_config)
test_config.project_dir = test_config.project_dir / 'foo' test_config.project_dir = test_config.project_dir / 'foo'
@ -67,11 +66,10 @@ def test_load_no_tokenizer_dir(test_config, tokenizer_mock, property_table):
factory.get_tokenizer_for_db(test_config) factory.get_tokenizer_for_db(test_config)
def test_load_missing_propoerty(temp_db_cursor, test_config, tokenizer_mock, property_table): def test_load_missing_propoerty(temp_db_cursor, test_config):
factory.create_tokenizer(test_config) factory.create_tokenizer(test_config)
temp_db_cursor.execute("TRUNCATE TABLE nominatim_properties") temp_db_cursor.execute("TRUNCATE TABLE nominatim_properties")
with pytest.raises(UsageError): with pytest.raises(UsageError):
factory.get_tokenizer_for_db(test_config) factory.get_tokenizer_for_db(test_config)

79
test/python/test_tokenizer_legacy.py
View File

@ -46,7 +46,7 @@ def tokenizer_factory(dsn, tmp_path, property_table):
@pytest.fixture @pytest.fixture
def tokenizer_setup(tokenizer_factory, test_config, monkeypatch, sql_preprocessor): def tokenizer_setup(tokenizer_factory, test_config, monkeypatch, sql_preprocessor):
monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None) monkeypatch.setattr(legacy_tokenizer, '_check_module', lambda m, c: None)
tok = tokenizer_factory() tok = tokenizer_factory()
tok.init_new_db(test_config) tok.init_new_db(test_config)
@ -60,7 +60,7 @@ def analyzer(tokenizer_factory, test_config, monkeypatch, sql_preprocessor,
RETURNS INTEGER AS $$ SELECT 342; $$ LANGUAGE SQL; RETURNS INTEGER AS $$ SELECT 342; $$ LANGUAGE SQL;
""") """)
monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None) monkeypatch.setattr(legacy_tokenizer, '_check_module', lambda m, c: None)
monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();') monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();')
tok = tokenizer_factory() tok = tokenizer_factory()
tok.init_new_db(test_config) tok.init_new_db(test_config)
@ -86,16 +86,6 @@ def create_postcode_id(temp_db_cursor):
$$ LANGUAGE SQL""") $$ LANGUAGE SQL""")
@pytest.fixture
def create_housenumbers(temp_db_cursor):
temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION create_housenumbers(
housenumbers TEXT[],
OUT tokens TEXT, OUT normtext TEXT)
AS $$
SELECT housenumbers::TEXT, array_to_string(housenumbers, ';')
$$ LANGUAGE SQL""")
@pytest.fixture @pytest.fixture
def make_keywords(temp_db_cursor, temp_db_with_extensions): def make_keywords(temp_db_cursor, temp_db_with_extensions):
temp_db_cursor.execute( temp_db_cursor.execute(
@ -105,7 +95,7 @@ def make_keywords(temp_db_cursor, temp_db_with_extensions):
def test_init_new(tokenizer_factory, test_config, monkeypatch, def test_init_new(tokenizer_factory, test_config, monkeypatch,
temp_db_conn, sql_preprocessor): temp_db_conn, sql_preprocessor):
monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', 'xxvv') monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', 'xxvv')
monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None) monkeypatch.setattr(legacy_tokenizer, '_check_module', lambda m, c: None)
tok = tokenizer_factory() tok = tokenizer_factory()
tok.init_new_db(test_config) tok.init_new_db(test_config)
@ -119,8 +109,7 @@ def test_init_new(tokenizer_factory, test_config, monkeypatch,
assert outfile.stat().st_mode == 33261 assert outfile.stat().st_mode == 33261
def test_init_module_load_failed(tokenizer_factory, test_config, def test_init_module_load_failed(tokenizer_factory, test_config):
monkeypatch, temp_db_conn):
tok = tokenizer_factory() tok = tokenizer_factory()
with pytest.raises(UsageError): with pytest.raises(UsageError):
@ -134,7 +123,7 @@ def test_init_module_custom(tokenizer_factory, test_config,
(module_dir/ 'nominatim.so').write_text('CUSTOM nomiantim.so') (module_dir/ 'nominatim.so').write_text('CUSTOM nomiantim.so')
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', str(module_dir)) monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', str(module_dir))
monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None) monkeypatch.setattr(legacy_tokenizer, '_check_module', lambda m, c: None)
tok = tokenizer_factory() tok = tokenizer_factory()
tok.init_new_db(test_config) tok.init_new_db(test_config)
@ -154,7 +143,7 @@ def test_update_sql_functions(sql_preprocessor, temp_db_conn,
tokenizer_factory, test_config, table_factory, tokenizer_factory, test_config, table_factory,
monkeypatch, temp_db_cursor): monkeypatch, temp_db_cursor):
monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '1133') monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '1133')
monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None) monkeypatch.setattr(legacy_tokenizer, '_check_module', lambda m, c: None)
tok = tokenizer_factory() tok = tokenizer_factory()
tok.init_new_db(test_config) tok.init_new_db(test_config)
monkeypatch.undo() monkeypatch.undo()
@ -174,7 +163,7 @@ def test_update_sql_functions(sql_preprocessor, temp_db_conn,
def test_migrate_database(tokenizer_factory, test_config, temp_db_conn, monkeypatch): def test_migrate_database(tokenizer_factory, test_config, temp_db_conn, monkeypatch):
monkeypatch.setattr(legacy_tokenizer, '_check_module' , lambda m, c: None) monkeypatch.setattr(legacy_tokenizer, '_check_module', lambda m, c: None)
tok = tokenizer_factory() tok = tokenizer_factory()
tok.migrate_database(test_config) tok.migrate_database(test_config)
@ -229,8 +218,7 @@ def test_update_special_phrase_empty_table(analyzer, word_table, make_standard_n
(' strasse', 'strasse', 'highway', 'primary', 'in'))) (' strasse', 'strasse', 'highway', 'primary', 'in')))
def test_update_special_phrase_delete_all(analyzer, word_table, temp_db_cursor, def test_update_special_phrase_delete_all(analyzer, word_table, make_standard_name):
make_standard_name):
word_table.add_special(' foo', 'foo', 'amenity', 'prison', 'in') word_table.add_special(' foo', 'foo', 'amenity', 'prison', 'in')
word_table.add_special(' bar', 'bar', 'highway', 'road', None) word_table.add_special(' bar', 'bar', 'highway', 'road', None)
@ -241,17 +229,15 @@ def test_update_special_phrase_delete_all(analyzer, word_table, temp_db_cursor,
assert word_table.count_special() == 0 assert word_table.count_special() == 0
def test_update_special_phrases_no_replace(analyzer, word_table, temp_db_cursor, def test_update_special_phrases_no_replace(analyzer, word_table, make_standard_name):
make_standard_name): word_table.add_special(' foo', 'foo', 'amenity', 'prison', 'in')
temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator) word_table.add_special(' bar', 'bar', 'highway', 'road', None)
VALUES (' foo', 'foo', 'amenity', 'prison', 'in'),
(' bar', 'bar', 'highway', 'road', null)""")
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""") assert word_table.count_special() == 2
analyzer.update_special_phrases([], False) analyzer.update_special_phrases([], False)
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""") assert word_table.count_special() == 2
def test_update_special_phrase_modify(analyzer, word_table, make_standard_name): def test_update_special_phrase_modify(analyzer, word_table, make_standard_name):
@ -273,41 +259,56 @@ def test_update_special_phrase_modify(analyzer, word_table, make_standard_name):
def test_process_place_names(analyzer, make_keywords): def test_process_place_names(analyzer, make_keywords):
info = analyzer.process_place({'name' : {'name' : 'Soft bAr', 'ref': '34'}}) info = analyzer.process_place({'name' : {'name' : 'Soft bAr', 'ref': '34'}})
assert info['names'] == '{1,2,3}' assert info['names'] == '{1,2,3}'
@pytest.mark.parametrize('pc', ['12345', 'AB 123', '34-345']) @pytest.mark.parametrize('pcode', ['12345', 'AB 123', '34-345'])
def test_process_place_postcode(analyzer, create_postcode_id, word_table, pc): def test_process_place_postcode(analyzer, create_postcode_id, word_table, pcode):
info = analyzer.process_place({'address': {'postcode' : pc}}) analyzer.process_place({'address': {'postcode' : pcode}})
assert word_table.get_postcodes() == {pc, } assert word_table.get_postcodes() == {pcode, }
@pytest.mark.parametrize('pc', ['12:23', 'ab;cd;f', '123;836']) @pytest.mark.parametrize('pcode', ['12:23', 'ab;cd;f', '123;836'])
def test_process_place_bad_postcode(analyzer, create_postcode_id, word_table, pc): def test_process_place_bad_postcode(analyzer, create_postcode_id, word_table, pcode):
info = analyzer.process_place({'address': {'postcode' : pc}}) analyzer.process_place({'address': {'postcode' : pcode}})
assert not word_table.get_postcodes() assert not word_table.get_postcodes()
@pytest.mark.parametrize('hnr', ['123a', '1', '101']) class TestHousenumberName:
def test_process_place_housenumbers_simple(analyzer, create_housenumbers, hnr):
@staticmethod
@pytest.fixture(autouse=True)
def setup_create_housenumbers(temp_db_cursor):
temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION create_housenumbers(
housenumbers TEXT[],
OUT tokens TEXT, OUT normtext TEXT)
AS $$
SELECT housenumbers::TEXT, array_to_string(housenumbers, ';')
$$ LANGUAGE SQL""")
@staticmethod
@pytest.mark.parametrize('hnr', ['123a', '1', '101'])
def test_process_place_housenumbers_simple(analyzer, hnr):
info = analyzer.process_place({'address': {'housenumber' : hnr}}) info = analyzer.process_place({'address': {'housenumber' : hnr}})
assert info['hnr'] == hnr assert info['hnr'] == hnr
assert info['hnr_tokens'].startswith("{") assert info['hnr_tokens'].startswith("{")
def test_process_place_housenumbers_lists(analyzer, create_housenumbers): @staticmethod
def test_process_place_housenumbers_lists(analyzer):
info = analyzer.process_place({'address': {'conscriptionnumber' : '1; 2;3'}}) info = analyzer.process_place({'address': {'conscriptionnumber' : '1; 2;3'}})
assert set(info['hnr'].split(';')) == set(('1', '2', '3')) assert set(info['hnr'].split(';')) == set(('1', '2', '3'))
def test_process_place_housenumbers_duplicates(analyzer, create_housenumbers): @staticmethod
def test_process_place_housenumbers_duplicates(analyzer):
info = analyzer.process_place({'address': {'housenumber' : '134', info = analyzer.process_place({'address': {'housenumber' : '134',
'conscriptionnumber' : '134', 'conscriptionnumber' : '134',
'streetnumber' : '99a'}}) 'streetnumber' : '99a'}})

139
test/python/test_tokenizer_legacy_icu.py
View File

@ -46,14 +46,14 @@ def db_prop(temp_db_conn):
return _get_db_property return _get_db_property
@pytest.fixture @pytest.fixture
def tokenizer_setup(tokenizer_factory, test_config, monkeypatch, sql_preprocessor): def tokenizer_setup(tokenizer_factory, test_config):
tok = tokenizer_factory() tok = tokenizer_factory()
tok.init_new_db(test_config) tok.init_new_db(test_config)
@pytest.fixture @pytest.fixture
def analyzer(tokenizer_factory, test_config, monkeypatch, sql_preprocessor, def analyzer(tokenizer_factory, test_config, monkeypatch,
word_table, temp_db_with_extensions, tmp_path): temp_db_with_extensions, tmp_path):
sql = tmp_path / 'sql' / 'tokenizer' / 'legacy_icu_tokenizer.sql' sql = tmp_path / 'sql' / 'tokenizer' / 'legacy_icu_tokenizer.sql'
sql.write_text("SELECT 'a';") sql.write_text("SELECT 'a';")
@ -74,17 +74,18 @@ def analyzer(tokenizer_factory, test_config, monkeypatch, sql_preprocessor,
@pytest.fixture @pytest.fixture
def getorcreate_term_id(temp_db_cursor): def getorcreate_term_id(temp_db_cursor):
temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_term_id(lookup_term TEXT) temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_term_id(lookup_term TEXT)
RETURNS INTEGER AS $$ SELECT nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""") RETURNS INTEGER AS $$
SELECT nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")
@pytest.fixture @pytest.fixture
def getorcreate_hnr_id(temp_db_cursor): def getorcreate_hnr_id(temp_db_cursor):
temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT) temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT)
RETURNS INTEGER AS $$ SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""") RETURNS INTEGER AS $$
SELECT -nextval('seq_word')::INTEGER; $$ LANGUAGE SQL""")
def test_init_new(tokenizer_factory, test_config, monkeypatch, db_prop, def test_init_new(tokenizer_factory, test_config, monkeypatch, db_prop):
sql_preprocessor, place_table, word_table):
monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();') monkeypatch.setenv('NOMINATIM_TERM_NORMALIZATION', ':: lower();')
tok = tokenizer_factory() tok = tokenizer_factory()
@ -105,10 +106,9 @@ def test_init_from_project(tokenizer_setup, tokenizer_factory):
assert tok.abbreviations is not None assert tok.abbreviations is not None
def test_update_sql_functions(temp_db_conn, db_prop, temp_db_cursor, def test_update_sql_functions(db_prop, temp_db_cursor,
tokenizer_factory, test_config, table_factory, tokenizer_factory, test_config, table_factory,
monkeypatch, monkeypatch):
sql_preprocessor, place_table, word_table):
monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '1133') monkeypatch.setenv('NOMINATIM_MAX_WORD_FREQUENCY', '1133')
tok = tokenizer_factory() tok = tokenizer_factory()
tok.init_new_db(test_config) tok.init_new_db(test_config)
@ -128,25 +128,25 @@ def test_update_sql_functions(temp_db_conn, db_prop, temp_db_cursor,
def test_make_standard_word(analyzer): def test_make_standard_word(analyzer):
with analyzer(abbr=(('STREET', 'ST'), ('tiny', 't'))) as a: with analyzer(abbr=(('STREET', 'ST'), ('tiny', 't'))) as anl:
assert a.make_standard_word('tiny street') == 'TINY ST' assert anl.make_standard_word('tiny street') == 'TINY ST'
with analyzer(abbr=(('STRASSE', 'STR'), ('STR', 'ST'))) as a: with analyzer(abbr=(('STRASSE', 'STR'), ('STR', 'ST'))) as anl:
assert a.make_standard_word('Hauptstrasse') == 'HAUPTST' assert anl.make_standard_word('Hauptstrasse') == 'HAUPTST'
def test_make_standard_hnr(analyzer): def test_make_standard_hnr(analyzer):
with analyzer(abbr=(('IV', '4'),)) as a: with analyzer(abbr=(('IV', '4'),)) as anl:
assert a._make_standard_hnr('345') == '345' assert anl._make_standard_hnr('345') == '345'
assert a._make_standard_hnr('iv') == 'IV' assert anl._make_standard_hnr('iv') == 'IV'
def test_update_postcodes_from_db_empty(analyzer, table_factory, word_table): def test_update_postcodes_from_db_empty(analyzer, table_factory, word_table):
table_factory('location_postcode', 'postcode TEXT', table_factory('location_postcode', 'postcode TEXT',
content=(('1234',), ('12 34',), ('AB23',), ('1234',))) content=(('1234',), ('12 34',), ('AB23',), ('1234',)))
with analyzer() as a: with analyzer() as anl:
a.update_postcodes_from_db() anl.update_postcodes_from_db()
assert word_table.count() == 3 assert word_table.count() == 3
assert word_table.get_postcodes() == {'1234', '12 34', 'AB23'} assert word_table.get_postcodes() == {'1234', '12 34', 'AB23'}
@ -158,119 +158,114 @@ def test_update_postcodes_from_db_add_and_remove(analyzer, table_factory, word_t
word_table.add_postcode(' 1234', '1234') word_table.add_postcode(' 1234', '1234')
word_table.add_postcode(' 5678', '5678') word_table.add_postcode(' 5678', '5678')
with analyzer() as a: with analyzer() as anl:
a.update_postcodes_from_db() anl.update_postcodes_from_db()
assert word_table.count() == 3 assert word_table.count() == 3
assert word_table.get_postcodes() == {'1234', '45BC', 'XX45'} assert word_table.get_postcodes() == {'1234', '45BC', 'XX45'}
def test_update_special_phrase_empty_table(analyzer, word_table, temp_db_cursor): def test_update_special_phrase_empty_table(analyzer, word_table):
with analyzer() as a: with analyzer() as anl:
a.update_special_phrases([ anl.update_special_phrases([
("König bei", "amenity", "royal", "near"), ("König bei", "amenity", "royal", "near"),
("Könige", "amenity", "royal", "-"), ("Könige", "amenity", "royal", "-"),
("street", "highway", "primary", "in") ("street", "highway", "primary", "in")
], True) ], True)
assert temp_db_cursor.row_set("""SELECT word_token, word, class, type, operator assert word_table.get_special() \
FROM word WHERE class != 'place'""") \ == {(' KÖNIG BEI', 'könig bei', 'amenity', 'royal', 'near'),
== set(((' KÖNIG BEI', 'könig bei', 'amenity', 'royal', 'near'),
(' KÖNIGE', 'könige', 'amenity', 'royal', None), (' KÖNIGE', 'könige', 'amenity', 'royal', None),
(' ST', 'street', 'highway', 'primary', 'in'))) (' ST', 'street', 'highway', 'primary', 'in')}
def test_update_special_phrase_delete_all(analyzer, word_table, temp_db_cursor): def test_update_special_phrase_delete_all(analyzer, word_table):
temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator) word_table.add_special(' FOO', 'foo', 'amenity', 'prison', 'in')
VALUES (' FOO', 'foo', 'amenity', 'prison', 'in'), word_table.add_special(' BAR', 'bar', 'highway', 'road', None)
(' BAR', 'bar', 'highway', 'road', null)""")
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""") assert word_table.count_special() == 2
with analyzer() as a: with analyzer() as anl:
a.update_special_phrases([], True) anl.update_special_phrases([], True)
assert 0 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""") assert word_table.count_special() == 0
def test_update_special_phrases_no_replace(analyzer, word_table, temp_db_cursor,): def test_update_special_phrases_no_replace(analyzer, word_table):
temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator) word_table.add_special(' FOO', 'foo', 'amenity', 'prison', 'in')
VALUES (' FOO', 'foo', 'amenity', 'prison', 'in'), word_table.add_special(' BAR', 'bar', 'highway', 'road', None)
(' BAR', 'bar', 'highway', 'road', null)""")
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""") assert word_table.count_special() == 2
with analyzer() as a: with analyzer() as anl:
a.update_special_phrases([], False) anl.update_special_phrases([], False)
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""") assert word_table.count_special() == 2
def test_update_special_phrase_modify(analyzer, word_table, temp_db_cursor): def test_update_special_phrase_modify(analyzer, word_table):
temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator) word_table.add_special(' FOO', 'foo', 'amenity', 'prison', 'in')
VALUES (' FOO', 'foo', 'amenity', 'prison', 'in'), word_table.add_special(' BAR', 'bar', 'highway', 'road', None)
(' BAR', 'bar', 'highway', 'road', null)""")
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""") assert word_table.count_special() == 2
with analyzer() as a: with analyzer() as anl:
a.update_special_phrases([ anl.update_special_phrases([
('prison', 'amenity', 'prison', 'in'), ('prison', 'amenity', 'prison', 'in'),
('bar', 'highway', 'road', '-'), ('bar', 'highway', 'road', '-'),
('garden', 'leisure', 'garden', 'near') ('garden', 'leisure', 'garden', 'near')
], True) ], True)
assert temp_db_cursor.row_set("""SELECT word_token, word, class, type, operator assert word_table.get_special() \
FROM word WHERE class != 'place'""") \ == {(' PRISON', 'prison', 'amenity', 'prison', 'in'),
== set(((' PRISON', 'prison', 'amenity', 'prison', 'in'),
(' BAR', 'bar', 'highway', 'road', None), (' BAR', 'bar', 'highway', 'road', None),
(' GARDEN', 'garden', 'leisure', 'garden', 'near'))) (' GARDEN', 'garden', 'leisure', 'garden', 'near')}
def test_process_place_names(analyzer, getorcreate_term_id): def test_process_place_names(analyzer, getorcreate_term_id):
with analyzer() as a: with analyzer() as anl:
info = a.process_place({'name' : {'name' : 'Soft bAr', 'ref': '34'}}) info = anl.process_place({'name' : {'name' : 'Soft bAr', 'ref': '34'}})
assert info['names'] == '{1,2,3,4,5,6}' assert info['names'] == '{1,2,3,4,5,6}'
@pytest.mark.parametrize('pc', ['12345', 'AB 123', '34-345']) @pytest.mark.parametrize('pcode', ['12345', 'AB 123', '34-345'])
def test_process_place_postcode(analyzer, word_table, pc): def test_process_place_postcode(analyzer, word_table, pcode):
with analyzer() as a: with analyzer() as anl:
info = a.process_place({'address': {'postcode' : pc}}) anl.process_place({'address': {'postcode' : pcode}})
assert word_table.get_postcodes() == {pc, } assert word_table.get_postcodes() == {pcode, }
@pytest.mark.parametrize('pc', ['12:23', 'ab;cd;f', '123;836']) @pytest.mark.parametrize('pcode', ['12:23', 'ab;cd;f', '123;836'])
def test_process_place_bad_postcode(analyzer, word_table, pc): def test_process_place_bad_postcode(analyzer, word_table, pcode):
with analyzer() as a: with analyzer() as anl:
info = a.process_place({'address': {'postcode' : pc}}) anl.process_place({'address': {'postcode' : pcode}})
assert not word_table.get_postcodes() assert not word_table.get_postcodes()
@pytest.mark.parametrize('hnr', ['123a', '1', '101']) @pytest.mark.parametrize('hnr', ['123a', '1', '101'])
def test_process_place_housenumbers_simple(analyzer, hnr, getorcreate_hnr_id): def test_process_place_housenumbers_simple(analyzer, hnr, getorcreate_hnr_id):
with analyzer() as a: with analyzer() as anl:
info = a.process_place({'address': {'housenumber' : hnr}}) info = anl.process_place({'address': {'housenumber' : hnr}})
assert info['hnr'] == hnr.upper() assert info['hnr'] == hnr.upper()
assert info['hnr_tokens'] == "{-1}" assert info['hnr_tokens'] == "{-1}"
def test_process_place_housenumbers_lists(analyzer, getorcreate_hnr_id): def test_process_place_housenumbers_lists(analyzer, getorcreate_hnr_id):
with analyzer() as a: with analyzer() as anl:
info = a.process_place({'address': {'conscriptionnumber' : '1; 2;3'}}) info = anl.process_place({'address': {'conscriptionnumber' : '1; 2;3'}})
assert set(info['hnr'].split(';')) == set(('1', '2', '3')) assert set(info['hnr'].split(';')) == set(('1', '2', '3'))
assert info['hnr_tokens'] == "{-1,-2,-3}" assert info['hnr_tokens'] == "{-1,-2,-3}"
def test_process_place_housenumbers_duplicates(analyzer, getorcreate_hnr_id): def test_process_place_housenumbers_duplicates(analyzer, getorcreate_hnr_id):
with analyzer() as a: with analyzer() as anl:
info = a.process_place({'address': {'housenumber' : '134', info = anl.process_place({'address': {'housenumber' : '134',
'conscriptionnumber' : '134', 'conscriptionnumber' : '134',
'streetnumber' : '99a'}}) 'streetnumber' : '99a'}})

30
test/python/test_tools_admin.py
View File

@ -3,39 +3,39 @@ Tests for maintenance and analysis functions.
""" """
import pytest import pytest
from nominatim.db.connection import connect
from nominatim.errors import UsageError from nominatim.errors import UsageError
from nominatim.tools import admin from nominatim.tools import admin
@pytest.fixture @pytest.fixture(autouse=True)
def db(temp_db, placex_table): def create_placex_table(placex_table):
with connect('dbname=' + temp_db) as conn: """ All tests in this module require the placex table to be set up.
yield conn """
def test_analyse_indexing_no_objects(db):
def test_analyse_indexing_no_objects(temp_db_conn):
with pytest.raises(UsageError): with pytest.raises(UsageError):
admin.analyse_indexing(db) admin.analyse_indexing(temp_db_conn)
@pytest.mark.parametrize("oid", ['1234', 'N123a', 'X123']) @pytest.mark.parametrize("oid", ['1234', 'N123a', 'X123'])
def test_analyse_indexing_bad_osmid(db, oid): def test_analyse_indexing_bad_osmid(temp_db_conn, oid):
with pytest.raises(UsageError): with pytest.raises(UsageError):
admin.analyse_indexing(db, osm_id=oid) admin.analyse_indexing(temp_db_conn, osm_id=oid)
def test_analyse_indexing_unknown_osmid(db): def test_analyse_indexing_unknown_osmid(temp_db_conn):
with pytest.raises(UsageError): with pytest.raises(UsageError):
admin.analyse_indexing(db, osm_id='W12345674') admin.analyse_indexing(temp_db_conn, osm_id='W12345674')
def test_analyse_indexing_with_place_id(db, temp_db_cursor): def test_analyse_indexing_with_place_id(temp_db_conn, temp_db_cursor):
temp_db_cursor.execute("INSERT INTO placex (place_id) VALUES(12345)") temp_db_cursor.execute("INSERT INTO placex (place_id) VALUES(12345)")
admin.analyse_indexing(db, place_id=12345) admin.analyse_indexing(temp_db_conn, place_id=12345)
def test_analyse_indexing_with_osm_id(db, temp_db_cursor): def test_analyse_indexing_with_osm_id(temp_db_conn, temp_db_cursor):
temp_db_cursor.execute("""INSERT INTO placex (place_id, osm_type, osm_id) temp_db_cursor.execute("""INSERT INTO placex (place_id, osm_type, osm_id)
VALUES(9988, 'N', 10000)""") VALUES(9988, 'N', 10000)""")
admin.analyse_indexing(db, osm_id='N10000') admin.analyse_indexing(temp_db_conn, osm_id='N10000')

39
test/python/test_tools_check_database.py
View File

@ -7,11 +7,11 @@ from nominatim.tools import check_database as chkdb
def test_check_database_unknown_db(def_config, monkeypatch): def test_check_database_unknown_db(def_config, monkeypatch):
monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'pgsql:dbname=fjgkhughwgh2423gsags') monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'pgsql:dbname=fjgkhughwgh2423gsags')
assert 1 == chkdb.check_database(def_config) assert chkdb.check_database(def_config) == 1
def test_check_database_fatal_test(def_config, temp_db): def test_check_database_fatal_test(def_config, temp_db):
assert 1 == chkdb.check_database(def_config) assert chkdb.check_database(def_config) == 1
def test_check_conection_good(temp_db_conn, def_config): def test_check_conection_good(temp_db_conn, def_config):
@ -23,8 +23,8 @@ def test_check_conection_bad(def_config):
assert chkdb.check_connection(badconn, def_config) == chkdb.CheckState.FATAL assert chkdb.check_connection(badconn, def_config) == chkdb.CheckState.FATAL
def test_check_placex_table_good(temp_db_cursor, temp_db_conn, def_config): def test_check_placex_table_good(table_factory, temp_db_conn, def_config):
temp_db_cursor.execute('CREATE TABLE placex (place_id int)') table_factory('placex')
assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.OK assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.OK
@ -32,14 +32,13 @@ def test_check_placex_table_bad(temp_db_conn, def_config):
assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.FATAL assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.FATAL
def test_check_placex_table_size_good(temp_db_cursor, temp_db_conn, def_config): def test_check_placex_table_size_good(table_factory, temp_db_conn, def_config):
temp_db_cursor.execute('CREATE TABLE placex (place_id int)') table_factory('placex', content=((1, ), (2, )))
temp_db_cursor.execute('INSERT INTO placex VALUES (1), (2)')
assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.OK assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.OK
def test_check_placex_table_size_bad(temp_db_cursor, temp_db_conn, def_config): def test_check_placex_table_size_bad(table_factory, temp_db_conn, def_config):
temp_db_cursor.execute('CREATE TABLE placex (place_id int)') table_factory('placex')
assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.FATAL assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.FATAL
@ -50,10 +49,11 @@ def test_check_tokenizer_missing(temp_db_conn, def_config, tmp_path):
@pytest.mark.parametrize("check_result,state", [(None, chkdb.CheckState.OK), @pytest.mark.parametrize("check_result,state", [(None, chkdb.CheckState.OK),
("Something wrong", chkdb.CheckState.FAIL)]) ("Something wrong", chkdb.CheckState.FAIL)])
def test_check_tokenizer(tokenizer_mock, temp_db_conn, def_config, monkeypatch, def test_check_tokenizer(temp_db_conn, def_config, monkeypatch,
check_result, state): check_result, state):
class _TestTokenizer: class _TestTokenizer:
def check_database(self): @staticmethod
def check_database():
return check_result return check_result
monkeypatch.setattr(chkdb.tokenizer_factory, 'get_tokenizer_for_db', monkeypatch.setattr(chkdb.tokenizer_factory, 'get_tokenizer_for_db',
@ -61,15 +61,15 @@ def test_check_tokenizer(tokenizer_mock, temp_db_conn, def_config, monkeypatch,
assert chkdb.check_tokenizer(temp_db_conn, def_config) == state assert chkdb.check_tokenizer(temp_db_conn, def_config) == state
def test_check_indexing_good(temp_db_cursor, temp_db_conn, def_config): def test_check_indexing_good(table_factory, temp_db_conn, def_config):
temp_db_cursor.execute('CREATE TABLE placex (place_id int, indexed_status smallint)') table_factory('placex', 'place_id int, indexed_status smallint',
temp_db_cursor.execute('INSERT INTO placex VALUES (1, 0), (2, 0)') content=((1, 0), (2, 0)))
assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.OK assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.OK
def test_check_indexing_bad(temp_db_cursor, temp_db_conn, def_config): def test_check_indexing_bad(table_factory, temp_db_conn, def_config):
temp_db_cursor.execute('CREATE TABLE placex (place_id int, indexed_status smallint)') table_factory('placex', 'place_id int, indexed_status smallint',
temp_db_cursor.execute('INSERT INTO placex VALUES (1, 0), (2, 2)') content=((1, 0), (2, 2)))
assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.FAIL assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.FAIL
@ -82,12 +82,12 @@ def test_check_database_indexes_valid(temp_db_conn, def_config):
def test_check_tiger_table_disabled(temp_db_conn, def_config, monkeypatch): def test_check_tiger_table_disabled(temp_db_conn, def_config, monkeypatch):
monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA' , 'no') monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA', 'no')
assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.NOT_APPLICABLE assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.NOT_APPLICABLE
def test_check_tiger_table_enabled(temp_db_cursor, temp_db_conn, def_config, monkeypatch): def test_check_tiger_table_enabled(temp_db_cursor, temp_db_conn, def_config, monkeypatch):
monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA' , 'yes') monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA', 'yes')
assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.FAIL assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.FAIL
temp_db_cursor.execute('CREATE TABLE location_property_tiger (place_id int)') temp_db_cursor.execute('CREATE TABLE location_property_tiger (place_id int)')
@ -95,4 +95,3 @@ def test_check_tiger_table_enabled(temp_db_cursor, temp_db_conn, def_config, mon
temp_db_cursor.execute('INSERT INTO location_property_tiger VALUES (1), (2)') temp_db_cursor.execute('INSERT INTO location_property_tiger VALUES (1), (2)')
assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.OK assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.OK

56
test/python/test_tools_database_import.py
View File

@ -1,10 +1,10 @@
""" """
Tests for functions to import a new database. Tests for functions to import a new database.
""" """
from pathlib import Path
import pytest import pytest
import psycopg2 import psycopg2
import sys
from pathlib import Path
from nominatim.tools import database_import from nominatim.tools import database_import
from nominatim.errors import UsageError from nominatim.errors import UsageError
@ -34,9 +34,9 @@ def test_setup_skeleton(src_dir, nonexistant_db, no_partitions):
try: try:
with conn.cursor() as cur: with conn.cursor() as cur:
cur.execute("SELECT distinct partition FROM country_name") cur.execute("SELECT distinct partition FROM country_name")
partitions = set([r[0] for r in list(cur)]) partitions = set((r[0] for r in list(cur)))
if no_partitions: if no_partitions:
assert partitions == set([0]) assert partitions == set((0, ))
else: else:
assert len(partitions) > 10 assert len(partitions) > 10
finally: finally:
@ -67,10 +67,11 @@ def test_create_db_missing_ro_user(nonexistant_db):
database_import.create_db('dbname=' + nonexistant_db, rouser='sdfwkjkjgdugu2;jgsafkljas;') database_import.create_db('dbname=' + nonexistant_db, rouser='sdfwkjkjgdugu2;jgsafkljas;')
def test_setup_extensions(temp_db_conn, temp_db_cursor): def test_setup_extensions(temp_db_conn, table_factory):
database_import.setup_extensions(temp_db_conn) database_import.setup_extensions(temp_db_conn)
temp_db_cursor.execute('CREATE TABLE t (h HSTORE, geom GEOMETRY(Geometry, 4326))') # Use table creation to check that hstore and geometry types are available.
table_factory('t', 'h HSTORE, geom GEOMETRY(Geometry, 4326)')
def test_setup_extensions_old_postgis(temp_db_conn, monkeypatch): def test_setup_extensions_old_postgis(temp_db_conn, monkeypatch):
@ -80,42 +81,36 @@ def test_setup_extensions_old_postgis(temp_db_conn, monkeypatch):
database_import.setup_extensions(temp_db_conn) database_import.setup_extensions(temp_db_conn)
def test_import_base_data(src_dir, temp_db, temp_db_cursor): def test_import_base_data(dsn, src_dir, temp_db_with_extensions, temp_db_cursor):
temp_db_cursor.execute('CREATE EXTENSION hstore') database_import.import_base_data(dsn, src_dir / 'data')
temp_db_cursor.execute('CREATE EXTENSION postgis')
database_import.import_base_data('dbname=' + temp_db, src_dir / 'data')
assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0 assert temp_db_cursor.table_rows('country_name') > 0
def test_import_base_data_ignore_partitions(src_dir, temp_db, temp_db_cursor): def test_import_base_data_ignore_partitions(dsn, src_dir, temp_db_with_extensions,
temp_db_cursor.execute('CREATE EXTENSION hstore') temp_db_cursor):
temp_db_cursor.execute('CREATE EXTENSION postgis') database_import.import_base_data(dsn, src_dir / 'data', ignore_partitions=True)
database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
ignore_partitions=True)
assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0 assert temp_db_cursor.table_rows('country_name') > 0
assert temp_db_cursor.scalar('SELECT count(*) FROM country_name WHERE partition != 0') == 0 assert temp_db_cursor.table_rows('country_name', where='partition != 0') == 0
def test_import_osm_data_simple(temp_db_cursor,osm2pgsql_options): def test_import_osm_data_simple(table_factory, osm2pgsql_options):
temp_db_cursor.execute('CREATE TABLE place (id INT)') table_factory('place', content=((1, ), ))
temp_db_cursor.execute('INSERT INTO place values (1)')
database_import.import_osm_data('file.pdf', osm2pgsql_options) database_import.import_osm_data('file.pdf', osm2pgsql_options)
def test_import_osm_data_simple_no_data(temp_db_cursor,osm2pgsql_options): def test_import_osm_data_simple_no_data(table_factory, osm2pgsql_options):
temp_db_cursor.execute('CREATE TABLE place (id INT)') table_factory('place')
with pytest.raises(UsageError, match='No data.*'): with pytest.raises(UsageError, match='No data.*'):
database_import.import_osm_data('file.pdf', osm2pgsql_options) database_import.import_osm_data('file.pdf', osm2pgsql_options)
def test_import_osm_data_drop(temp_db_conn, temp_db_cursor, tmp_path, osm2pgsql_options): def test_import_osm_data_drop(table_factory, temp_db_conn, tmp_path, osm2pgsql_options):
temp_db_cursor.execute('CREATE TABLE place (id INT)') table_factory('place', content=((1, ), ))
temp_db_cursor.execute('CREATE TABLE planet_osm_nodes (id INT)') table_factory('planet_osm_nodes')
temp_db_cursor.execute('INSERT INTO place values (1)')
flatfile = tmp_path / 'flatfile' flatfile = tmp_path / 'flatfile'
flatfile.write_text('touch') flatfile.write_text('touch')
@ -128,9 +123,8 @@ def test_import_osm_data_drop(temp_db_conn, temp_db_cursor, tmp_path, osm2pgsql_
assert not temp_db_conn.table_exists('planet_osm_nodes') assert not temp_db_conn.table_exists('planet_osm_nodes')
def test_import_osm_data_default_cache(temp_db_cursor,osm2pgsql_options): def test_import_osm_data_default_cache(table_factory, osm2pgsql_options):
temp_db_cursor.execute('CREATE TABLE place (id INT)') table_factory('place', content=((1, ), ))
temp_db_cursor.execute('INSERT INTO place values (1)')
osm2pgsql_options['osm2pgsql_cache'] = 0 osm2pgsql_options['osm2pgsql_cache'] = 0
@ -153,7 +147,7 @@ def test_truncate_database_tables(temp_db_conn, temp_db_cursor, table_factory):
@pytest.mark.parametrize("threads", (1, 5)) @pytest.mark.parametrize("threads", (1, 5))
def test_load_data(dsn, src_dir, place_row, placex_table, osmline_table, def test_load_data(dsn, place_row, placex_table, osmline_table,
word_table, temp_db_cursor, threads): word_table, temp_db_cursor, threads):
for func in ('precompute_words', 'getorcreate_housenumber_id', 'make_standard_name'): for func in ('precompute_words', 'getorcreate_housenumber_id', 'make_standard_name'):
temp_db_cursor.execute("""CREATE FUNCTION {} (src TEXT) temp_db_cursor.execute("""CREATE FUNCTION {} (src TEXT)

101
test/python/test_tools_exec_utils.py
View File

@ -3,14 +3,19 @@ Tests for tools.exec_utils module.
""" """
from pathlib import Path from pathlib import Path
import subprocess import subprocess
import tempfile
import pytest import pytest
import nominatim.tools.exec_utils as exec_utils import nominatim.tools.exec_utils as exec_utils
@pytest.fixture class TestRunLegacyScript:
def nominatim_env(tmp_phplib_dir, def_config):
@pytest.fixture(autouse=True)
def setup_nominatim_env(self, tmp_path, def_config):
tmp_phplib_dir = tmp_path / 'phplib'
tmp_phplib_dir.mkdir()
(tmp_phplib_dir / 'admin').mkdir()
class _NominatimEnv: class _NominatimEnv:
config = def_config config = def_config
phplib_dir = tmp_phplib_dir phplib_dir = tmp_phplib_dir
@ -21,79 +26,71 @@ def nominatim_env(tmp_phplib_dir, def_config):
module_dir = 'module' module_dir = 'module'
osm2pgsql_path = 'osm2pgsql' osm2pgsql_path = 'osm2pgsql'
return _NominatimEnv self.testenv = _NominatimEnv
@pytest.fixture
def test_script(nominatim_env): def mk_script(self, code):
def _create_file(code): codefile = self.testenv.phplib_dir / 'admin' / 't.php'
with (nominatim_env.phplib_dir / 'admin' / 't.php').open(mode='w') as fd: codefile.write_text('<?php\n' + code + '\n')
fd.write('<?php\n')
fd.write(code + '\n')
return 't.php' return 't.php'
return _create_file
@pytest.fixture(params=[0, 1, 15, 255]) @pytest.mark.parametrize("return_code", (0, 1, 15, 255))
def return_code(request): def test_run_legacy_return_exit_code(self, return_code):
return request.param fname = self.mk_script('exit({});'.format(return_code))
assert return_code == \
### run_legacy_script exec_utils.run_legacy_script(fname, nominatim_env=self.testenv)
def test_run_legacy_return_exit_code(nominatim_env, test_script, return_code):
fname = test_script('exit({});'.format(return_code))
assert return_code == exec_utils.run_legacy_script(fname,
nominatim_env=nominatim_env)
def test_run_legacy_return_throw_on_fail(nominatim_env, test_script): def test_run_legacy_return_throw_on_fail(self):
fname = test_script('exit(11);') fname = self.mk_script('exit(11);')
with pytest.raises(subprocess.CalledProcessError): with pytest.raises(subprocess.CalledProcessError):
exec_utils.run_legacy_script(fname, nominatim_env=nominatim_env, exec_utils.run_legacy_script(fname, nominatim_env=self.testenv,
throw_on_fail=True) throw_on_fail=True)
def test_run_legacy_return_dont_throw_on_success(nominatim_env, test_script): def test_run_legacy_return_dont_throw_on_success(self):
fname = test_script('exit(0);') fname = self.mk_script('exit(0);')
assert 0 == exec_utils.run_legacy_script(fname, nominatim_env=nominatim_env, assert exec_utils.run_legacy_script(fname, nominatim_env=self.testenv,
throw_on_fail=True) throw_on_fail=True) == 0
def test_run_legacy_use_given_module_path(nominatim_env, test_script): def test_run_legacy_use_given_module_path(self):
fname = test_script("exit($_SERVER['NOMINATIM_DATABASE_MODULE_PATH'] == '' ? 0 : 23);") fname = self.mk_script("exit($_SERVER['NOMINATIM_DATABASE_MODULE_PATH'] == '' ? 0 : 23);")
assert 0 == exec_utils.run_legacy_script(fname, nominatim_env=nominatim_env) assert exec_utils.run_legacy_script(fname, nominatim_env=self.testenv) == 0
def test_run_legacy_do_not_overwrite_module_path(nominatim_env, test_script, monkeypatch): def test_run_legacy_do_not_overwrite_module_path(self, monkeypatch):
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', 'other') monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', 'other')
fname = test_script("exit($_SERVER['NOMINATIM_DATABASE_MODULE_PATH'] == 'other' ? 0 : 1);") fname = self.mk_script(
"exit($_SERVER['NOMINATIM_DATABASE_MODULE_PATH'] == 'other' ? 0 : 1);")
assert 0 == exec_utils.run_legacy_script(fname, nominatim_env=nominatim_env) assert exec_utils.run_legacy_script(fname, nominatim_env=self.testenv) == 0
### run_api_script
@pytest.fixture class TestRunApiScript:
def tmp_project_dir():
with tempfile.TemporaryDirectory() as tempd: @staticmethod
project_dir = Path(tempd) @pytest.fixture(autouse=True)
webdir = project_dir / 'website' def setup_project_dir(tmp_path):
webdir = tmp_path / 'website'
webdir.mkdir() webdir.mkdir()
(webdir / 'test.php').write_text("<?php\necho 'OK\n';")
with (webdir / 'test.php').open(mode='w') as fd:
fd.write("<?php\necho 'OK\n';")
yield project_dir @staticmethod
def test_run_api(tmp_path):
assert exec_utils.run_api_script('test', tmp_path) == 0
def test_run_api(tmp_project_dir): @staticmethod
assert 0 == exec_utils.run_api_script('test', tmp_project_dir) def test_run_api_execution_error(tmp_path):
assert exec_utils.run_api_script('badname', tmp_path) != 0
def test_run_api_execution_error(tmp_project_dir): @staticmethod
assert 0 != exec_utils.run_api_script('badname', tmp_project_dir) def test_run_api_with_extra_env(tmp_path):
extra_env = dict(SCRIPT_FILENAME=str(tmp_path / 'website' / 'test.php'))
def test_run_api_with_extra_env(tmp_project_dir): assert exec_utils.run_api_script('badname', tmp_path, extra_env=extra_env) == 0
extra_env = dict(SCRIPT_FILENAME=str(tmp_project_dir / 'website' / 'test.php'))
assert 0 == exec_utils.run_api_script('badname', tmp_project_dir,
extra_env=extra_env)
### run_osm2pgsql ### run_osm2pgsql

6
test/python/test_tools_freeze.py
View File

@ -1,8 +1,6 @@
""" """
Tests for freeze functions (removing unused database parts). Tests for freeze functions (removing unused database parts).
""" """
import pytest
from nominatim.tools import freeze from nominatim.tools import freeze
NOMINATIM_RUNTIME_TABLES = [ NOMINATIM_RUNTIME_TABLES = [
@ -22,9 +20,9 @@ NOMINATIM_DROP_TABLES = [
'wikipedia_article', 'wikipedia_redirect' 'wikipedia_article', 'wikipedia_redirect'
] ]
def test_drop_tables(temp_db_conn, temp_db_cursor): def test_drop_tables(temp_db_conn, temp_db_cursor, table_factory):
for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES: for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES:
temp_db_cursor.execute('CREATE TABLE {} (id int)'.format(table)) table_factory(table)
freeze.drop_update_tables(temp_db_conn) freeze.drop_update_tables(temp_db_conn)

194
test/python/test_tools_import_special_phrases.py
View File

@ -2,24 +2,68 @@
Tests for import special phrases methods Tests for import special phrases methods
of the class SPImporter. of the class SPImporter.
""" """
from nominatim.errors import UsageError
from pathlib import Path
import tempfile
from shutil import copyfile from shutil import copyfile
import pytest import pytest
from nominatim.tools.special_phrases.sp_importer import SPImporter from nominatim.tools.special_phrases.sp_importer import SPImporter
from nominatim.tools.special_phrases.sp_wiki_loader import SPWikiLoader from nominatim.tools.special_phrases.sp_wiki_loader import SPWikiLoader
from nominatim.tools.special_phrases.sp_csv_loader import SPCsvLoader
from nominatim.tools.special_phrases.special_phrase import SpecialPhrase from nominatim.tools.special_phrases.special_phrase import SpecialPhrase
from nominatim.errors import UsageError
TEST_BASE_DIR = Path(__file__) / '..' / '..' from cursor import CursorForTesting
def test_fetch_existing_place_classtype_tables(sp_importer, temp_db_cursor): @pytest.fixture
def testfile_dir(src_dir):
return src_dir / 'test' / 'testfiles'
@pytest.fixture
def sp_importer(temp_db_conn, def_config, temp_phplib_dir_with_migration):
"""
Return an instance of SPImporter.
"""
loader = SPWikiLoader(def_config, ['en'])
return SPImporter(def_config, temp_phplib_dir_with_migration, temp_db_conn, loader)
@pytest.fixture
def temp_phplib_dir_with_migration(src_dir, tmp_path):
"""
Return temporary phpdir with migration subdirectory and
PhraseSettingsToJson.php script inside.
"""
migration_file = (src_dir / 'lib-php' / 'migration' / 'PhraseSettingsToJson.php').resolve()
phpdir = tmp_path / 'tempphp'
phpdir.mkdir()
(phpdir / 'migration').mkdir()
migration_dest_path = (phpdir / 'migration' / 'PhraseSettingsToJson.php').resolve()
copyfile(str(migration_file), str(migration_dest_path))
return phpdir
@pytest.fixture
def xml_wiki_content(src_dir):
"""
return the content of the static xml test file.
"""
xml_test_content = src_dir / 'test' / 'testdata' / 'special_phrases_test_content.txt'
return xml_test_content.read_text()
@pytest.fixture
def default_phrases(table_factory):
table_factory('place_classtype_testclasstypetable_to_delete')
table_factory('place_classtype_testclasstypetable_to_keep')
def test_fetch_existing_place_classtype_tables(sp_importer, table_factory):
""" """
Check for the fetch_existing_place_classtype_tables() method. Check for the fetch_existing_place_classtype_tables() method.
It should return the table just created. It should return the table just created.
""" """
temp_db_cursor.execute('CREATE TABLE place_classtype_testclasstypetable()') table_factory('place_classtype_testclasstypetable')
sp_importer._fetch_existing_place_classtype_tables() sp_importer._fetch_existing_place_classtype_tables()
contained_table = sp_importer.table_phrases_to_delete.pop() contained_table = sp_importer.table_phrases_to_delete.pop()
@ -46,19 +90,18 @@ def test_load_white_and_black_lists(sp_importer):
assert isinstance(black_list, dict) and isinstance(white_list, dict) assert isinstance(black_list, dict) and isinstance(white_list, dict)
def test_convert_php_settings(sp_importer): def test_convert_php_settings(sp_importer, testfile_dir, tmp_path):
""" """
Test that _convert_php_settings_if_needed() convert the given Test that _convert_php_settings_if_needed() convert the given
php file to a json file. php file to a json file.
""" """
php_file = (TEST_BASE_DIR / 'testfiles' / 'phrase_settings.php').resolve() php_file = (testfile_dir / 'phrase_settings.php').resolve()
with tempfile.TemporaryDirectory() as temp_dir: temp_settings = (tmp_path / 'phrase_settings.php').resolve()
temp_settings = (Path(temp_dir) / 'phrase_settings.php').resolve()
copyfile(php_file, temp_settings) copyfile(php_file, temp_settings)
sp_importer._convert_php_settings_if_needed(temp_settings) sp_importer._convert_php_settings_if_needed(temp_settings)
assert (Path(temp_dir) / 'phrase_settings.json').is_file() assert (tmp_path / 'phrase_settings.json').is_file()
def test_convert_settings_wrong_file(sp_importer): def test_convert_settings_wrong_file(sp_importer):
""" """
@ -68,30 +111,31 @@ def test_convert_settings_wrong_file(sp_importer):
with pytest.raises(UsageError, match='random_file is not a valid file.'): with pytest.raises(UsageError, match='random_file is not a valid file.'):
sp_importer._convert_php_settings_if_needed('random_file') sp_importer._convert_php_settings_if_needed('random_file')
def test_convert_settings_json_already_exist(sp_importer): def test_convert_settings_json_already_exist(sp_importer, testfile_dir):
""" """
Test that if we give to '_convert_php_settings_if_needed' a php file path Test that if we give to '_convert_php_settings_if_needed' a php file path
and that a the corresponding json file already exists, it is returned. and that a the corresponding json file already exists, it is returned.
""" """
php_file = (TEST_BASE_DIR / 'testfiles' / 'phrase_settings.php').resolve() php_file = (testfile_dir / 'phrase_settings.php').resolve()
json_file = (TEST_BASE_DIR / 'testfiles' / 'phrase_settings.json').resolve() json_file = (testfile_dir / 'phrase_settings.json').resolve()
returned = sp_importer._convert_php_settings_if_needed(php_file) returned = sp_importer._convert_php_settings_if_needed(php_file)
assert returned == json_file assert returned == json_file
def test_convert_settings_giving_json(sp_importer): def test_convert_settings_giving_json(sp_importer, testfile_dir):
""" """
Test that if we give to '_convert_php_settings_if_needed' a json file path Test that if we give to '_convert_php_settings_if_needed' a json file path
the same path is directly returned the same path is directly returned
""" """
json_file = (TEST_BASE_DIR / 'testfiles' / 'phrase_settings.json').resolve() json_file = (testfile_dir / 'phrase_settings.json').resolve()
returned = sp_importer._convert_php_settings_if_needed(json_file) returned = sp_importer._convert_php_settings_if_needed(json_file)
assert returned == json_file assert returned == json_file
def test_create_place_classtype_indexes(temp_db_conn, sp_importer): def test_create_place_classtype_indexes(temp_db_with_extensions, temp_db_conn,
table_factory, sp_importer):
""" """
Test that _create_place_classtype_indexes() create the Test that _create_place_classtype_indexes() create the
place_id index and centroid index on the right place_class_type table. place_id index and centroid index on the right place_class_type table.
@ -100,9 +144,7 @@ def test_create_place_classtype_indexes(temp_db_conn, sp_importer):
phrase_type = 'type' phrase_type = 'type'
table_name = 'place_classtype_{}_{}'.format(phrase_class, phrase_type) table_name = 'place_classtype_{}_{}'.format(phrase_class, phrase_type)
with temp_db_conn.cursor() as temp_db_cursor: table_factory(table_name, 'place_id BIGINT, centroid GEOMETRY')
temp_db_cursor.execute("CREATE EXTENSION postgis;")
temp_db_cursor.execute('CREATE TABLE {}(place_id BIGINT, centroid GEOMETRY)'.format(table_name))
sp_importer._create_place_classtype_indexes('', phrase_class, phrase_type) sp_importer._create_place_classtype_indexes('', phrase_class, phrase_type)
@ -119,7 +161,7 @@ def test_create_place_classtype_table(temp_db_conn, placex_table, sp_importer):
assert check_table_exist(temp_db_conn, phrase_class, phrase_type) assert check_table_exist(temp_db_conn, phrase_class, phrase_type)
def test_grant_access_to_web_user(temp_db_conn, def_config, sp_importer): def test_grant_access_to_web_user(temp_db_conn, table_factory, def_config, sp_importer):
""" """
Test that _grant_access_to_webuser() give Test that _grant_access_to_webuser() give
right access to the web user. right access to the web user.
@ -128,8 +170,7 @@ def test_grant_access_to_web_user(temp_db_conn, def_config, sp_importer):
phrase_type = 'type' phrase_type = 'type'
table_name = 'place_classtype_{}_{}'.format(phrase_class, phrase_type) table_name = 'place_classtype_{}_{}'.format(phrase_class, phrase_type)
with temp_db_conn.cursor() as temp_db_cursor: table_factory(table_name)
temp_db_cursor.execute('CREATE TABLE {}()'.format(table_name))
sp_importer._grant_access_to_webuser(phrase_class, phrase_type) sp_importer._grant_access_to_webuser(phrase_class, phrase_type)
@ -165,7 +206,6 @@ def test_remove_non_existent_tables_from_db(sp_importer, default_phrases,
place_classtype tables contained in table_phrases_to_delete should place_classtype tables contained in table_phrases_to_delete should
be deleted. be deleted.
""" """
with temp_db_conn.cursor() as temp_db_cursor:
sp_importer.table_phrases_to_delete = { sp_importer.table_phrases_to_delete = {
'place_classtype_testclasstypetable_to_delete' 'place_classtype_testclasstypetable_to_delete'
} }
@ -179,15 +219,16 @@ def test_remove_non_existent_tables_from_db(sp_importer, default_phrases,
sp_importer._remove_non_existent_tables_from_db() sp_importer._remove_non_existent_tables_from_db()
temp_db_cursor.execute(query_tables) # Changes are not committed yet. Use temp_db_conn for checking results.
tables_result = temp_db_cursor.fetchall() with temp_db_conn.cursor(cursor_factory=CursorForTesting) as cur:
assert (len(tables_result) == 1 and assert cur.row_set(query_tables) \
tables_result[0][0] == 'place_classtype_testclasstypetable_to_keep' == {('place_classtype_testclasstypetable_to_keep', )}
)
@pytest.mark.parametrize("should_replace", [(True), (False)]) @pytest.mark.parametrize("should_replace", [(True), (False)])
def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer, def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
placex_table, tokenizer_mock, should_replace): placex_table, table_factory, tokenizer_mock,
xml_wiki_content, should_replace):
""" """
Check that the main import_phrases() method is well executed. Check that the main import_phrases() method is well executed.
It should create the place_classtype table, the place_id and centroid indexes, It should create the place_classtype table, the place_id and centroid indexes,
@ -197,13 +238,11 @@ def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
""" """
#Add some data to the database before execution in order to test #Add some data to the database before execution in order to test
#what is deleted and what is preserved. #what is deleted and what is preserved.
with temp_db_conn.cursor() as temp_db_cursor: table_factory('place_classtype_amenity_animal_shelter')
temp_db_cursor.execute(""" table_factory('place_classtype_wrongclass_wrongtype')
CREATE TABLE place_classtype_amenity_animal_shelter();
CREATE TABLE place_classtype_wrongclass_wrongtype();""")
monkeypatch.setattr('nominatim.tools.special_phrases.sp_wiki_loader.SPWikiLoader._get_wiki_content', monkeypatch.setattr('nominatim.tools.special_phrases.sp_wiki_loader.SPWikiLoader._get_wiki_content',
mock_get_wiki_content) lambda self, lang: xml_wiki_content)
tokenizer = tokenizer_mock() tokenizer = tokenizer_mock()
sp_importer.import_phrases(tokenizer, should_replace) sp_importer.import_phrases(tokenizer, should_replace)
@ -220,65 +259,17 @@ def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
if should_replace: if should_replace:
assert not check_table_exist(temp_db_conn, 'wrong_class', 'wrong_type') assert not check_table_exist(temp_db_conn, 'wrong_class', 'wrong_type')
#Format (query, should_return_something_bool) use to easily execute all asserts assert temp_db_conn.table_exists('place_classtype_amenity_animal_shelter')
queries_tests = set()
#Used to check that correct place_classtype table already in the datase before is still there.
query_existing_table = """
SELECT table_name
FROM information_schema.tables
WHERE table_schema='public'
AND table_name = 'place_classtype_amenity_animal_shelter';
"""
queries_tests.add((query_existing_table, True))
#Used to check that wrong place_classtype table was deleted from the database.
query_wrong_table = """
SELECT table_name
FROM information_schema.tables
WHERE table_schema='public'
AND table_name = 'place_classtype_wrongclass_wrongtype';
"""
if should_replace: if should_replace:
queries_tests.add((query_wrong_table, False)) assert not temp_db_conn.table_exists('place_classtype_wrongclass_wrongtype')
with temp_db_conn.cursor() as temp_db_cursor:
for query in queries_tests:
temp_db_cursor.execute(query[0])
if (query[1] == True):
assert temp_db_cursor.fetchone()
else:
assert not temp_db_cursor.fetchone()
def mock_get_wiki_content(self, lang):
"""
Mock the _get_wiki_content() method to return
static xml test file content.
"""
return get_test_xml_wiki_content()
def get_test_xml_wiki_content():
"""
return the content of the static xml test file.
"""
xml_test_content_path = (TEST_BASE_DIR / 'testdata' / 'special_phrases_test_content.txt').resolve()
with open(xml_test_content_path) as xml_content_reader:
return xml_content_reader.read()
def check_table_exist(temp_db_conn, phrase_class, phrase_type): def check_table_exist(temp_db_conn, phrase_class, phrase_type):
""" """
Verify that the place_classtype table exists for the given Verify that the place_classtype table exists for the given
phrase_class and phrase_type. phrase_class and phrase_type.
""" """
table_name = 'place_classtype_{}_{}'.format(phrase_class, phrase_type) return temp_db_conn.table_exists('place_classtype_{}_{}'.format(phrase_class, phrase_type))
with temp_db_conn.cursor() as temp_db_cursor:
temp_db_cursor.execute("""
SELECT *
FROM information_schema.tables
WHERE table_type='BASE TABLE'
AND table_name='{}'""".format(table_name))
return temp_db_cursor.fetchone()
def check_grant_access(temp_db_conn, user, phrase_class, phrase_type): def check_grant_access(temp_db_conn, user, phrase_class, phrase_type):
""" """
@ -307,32 +298,3 @@ def check_placeid_and_centroid_indexes(temp_db_conn, phrase_class, phrase_type):
and and
temp_db_conn.index_exists(index_prefix + 'place_id') temp_db_conn.index_exists(index_prefix + 'place_id')
) )
@pytest.fixture
def sp_importer(temp_db_conn, def_config, temp_phplib_dir_with_migration):
"""
Return an instance of SPImporter.
"""
loader = SPWikiLoader(def_config, ['en'])
return SPImporter(def_config, temp_phplib_dir_with_migration, temp_db_conn, loader)
@pytest.fixture
def temp_phplib_dir_with_migration():
"""
Return temporary phpdir with migration subdirectory and
PhraseSettingsToJson.php script inside.
"""
migration_file = (TEST_BASE_DIR / '..' / 'lib-php' / 'migration'
/ 'PhraseSettingsToJson.php').resolve()
with tempfile.TemporaryDirectory() as phpdir:
(Path(phpdir) / 'migration').mkdir()
migration_dest_path = (Path(phpdir) / 'migration' / 'PhraseSettingsToJson.php').resolve()
copyfile(migration_file, migration_dest_path)
yield Path(phpdir)
@pytest.fixture
def default_phrases(temp_db_cursor):
temp_db_cursor.execute("""
CREATE TABLE place_classtype_testclasstypetable_to_delete();
CREATE TABLE place_classtype_testclasstypetable_to_keep();""")

24
test/python/test_tools_postcodes.py
View File

@ -58,13 +58,13 @@ def postcode_table(temp_db_conn, placex_table, word_table):
return MockPostcodeTable(temp_db_conn) return MockPostcodeTable(temp_db_conn)
def test_import_postcodes_empty(dsn, postcode_table, tmp_path, tokenizer): def test_postcodes_empty(dsn, postcode_table, tmp_path, tokenizer):
postcodes.update_postcodes(dsn, tmp_path, tokenizer) postcodes.update_postcodes(dsn, tmp_path, tokenizer)
assert not postcode_table.row_set assert not postcode_table.row_set
def test_import_postcodes_add_new(dsn, placex_table, postcode_table, tmp_path, tokenizer): def test_postcodes_add_new(dsn, placex_table, postcode_table, tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)', placex_table.add(country='xx', geom='POINT(10 12)',
address=dict(postcode='9486')) address=dict(postcode='9486'))
postcode_table.add('yy', '9486', 99, 34) postcode_table.add('yy', '9486', 99, 34)
@ -74,7 +74,8 @@ def test_import_postcodes_add_new(dsn, placex_table, postcode_table, tmp_path, t
assert postcode_table.row_set == {('xx', '9486', 10, 12), } assert postcode_table.row_set == {('xx', '9486', 10, 12), }
def test_import_postcodes_replace_coordinates(dsn, placex_table, postcode_table, tmp_path, tokenizer): def test_postcodes_replace_coordinates(dsn, placex_table, postcode_table,
tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)', placex_table.add(country='xx', geom='POINT(10 12)',
address=dict(postcode='AB 4511')) address=dict(postcode='AB 4511'))
postcode_table.add('xx', 'AB 4511', 99, 34) postcode_table.add('xx', 'AB 4511', 99, 34)
@ -84,7 +85,8 @@ def test_import_postcodes_replace_coordinates(dsn, placex_table, postcode_table,
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)} assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
def test_import_postcodes_replace_coordinates_close(dsn, placex_table, postcode_table, tmp_path, tokenizer): def test_postcodes_replace_coordinates_close(dsn, placex_table, postcode_table,
tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)', placex_table.add(country='xx', geom='POINT(10 12)',
address=dict(postcode='AB 4511')) address=dict(postcode='AB 4511'))
postcode_table.add('xx', 'AB 4511', 10, 11.99999) postcode_table.add('xx', 'AB 4511', 10, 11.99999)
@ -94,7 +96,7 @@ def test_import_postcodes_replace_coordinates_close(dsn, placex_table, postcode_
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 11.99999)} assert postcode_table.row_set == {('xx', 'AB 4511', 10, 11.99999)}
def test_import_postcodes_remove(dsn, placex_table, postcode_table, tmp_path, tokenizer): def test_postcodes_remove(dsn, placex_table, postcode_table, tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)', placex_table.add(country='xx', geom='POINT(10 12)',
address=dict(postcode='AB 4511')) address=dict(postcode='AB 4511'))
postcode_table.add('xx', 'badname', 10, 12) postcode_table.add('xx', 'badname', 10, 12)
@ -104,7 +106,7 @@ def test_import_postcodes_remove(dsn, placex_table, postcode_table, tmp_path, to
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)} assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
def test_import_postcodes_ignore_empty_country(dsn, placex_table, postcode_table, tmp_path, tokenizer): def test_postcodes_ignore_empty_country(dsn, placex_table, postcode_table, tmp_path, tokenizer):
placex_table.add(country=None, geom='POINT(10 12)', placex_table.add(country=None, geom='POINT(10 12)',
address=dict(postcode='AB 4511')) address=dict(postcode='AB 4511'))
@ -113,7 +115,7 @@ def test_import_postcodes_ignore_empty_country(dsn, placex_table, postcode_table
assert not postcode_table.row_set assert not postcode_table.row_set
def test_import_postcodes_remove_all(dsn, placex_table, postcode_table, tmp_path, tokenizer): def test_postcodes_remove_all(dsn, postcode_table, tmp_path, tokenizer):
postcode_table.add('ch', '5613', 10, 12) postcode_table.add('ch', '5613', 10, 12)
postcodes.update_postcodes(dsn, tmp_path, tokenizer) postcodes.update_postcodes(dsn, tmp_path, tokenizer)
@ -121,7 +123,7 @@ def test_import_postcodes_remove_all(dsn, placex_table, postcode_table, tmp_path
assert not postcode_table.row_set assert not postcode_table.row_set
def test_import_postcodes_multi_country(dsn, placex_table, postcode_table, tmp_path, tokenizer): def test_postcodes_multi_country(dsn, placex_table, postcode_table, tmp_path, tokenizer):
placex_table.add(country='de', geom='POINT(10 12)', placex_table.add(country='de', geom='POINT(10 12)',
address=dict(postcode='54451')) address=dict(postcode='54451'))
placex_table.add(country='cc', geom='POINT(100 56)', placex_table.add(country='cc', geom='POINT(100 56)',
@ -140,7 +142,7 @@ def test_import_postcodes_multi_country(dsn, placex_table, postcode_table, tmp_p
@pytest.mark.parametrize("gzipped", [True, False]) @pytest.mark.parametrize("gzipped", [True, False])
def test_import_postcodes_extern(dsn, placex_table, postcode_table, tmp_path, def test_postcodes_extern(dsn, placex_table, postcode_table, tmp_path,
tokenizer, gzipped): tokenizer, gzipped):
placex_table.add(country='xx', geom='POINT(10 12)', placex_table.add(country='xx', geom='POINT(10 12)',
address=dict(postcode='AB 4511')) address=dict(postcode='AB 4511'))
@ -158,7 +160,7 @@ def test_import_postcodes_extern(dsn, placex_table, postcode_table, tmp_path,
('xx', 'CD 4511', -10, -5)} ('xx', 'CD 4511', -10, -5)}
def test_import_postcodes_extern_bad_column(dsn, placex_table, postcode_table, def test_postcodes_extern_bad_column(dsn, placex_table, postcode_table,
tmp_path, tokenizer): tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)', placex_table.add(country='xx', geom='POINT(10 12)',
address=dict(postcode='AB 4511')) address=dict(postcode='AB 4511'))
@ -171,7 +173,7 @@ def test_import_postcodes_extern_bad_column(dsn, placex_table, postcode_table,
assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)} assert postcode_table.row_set == {('xx', 'AB 4511', 10, 12)}
def test_import_postcodes_extern_bad_number(dsn, placex_table, postcode_table, def test_postcodes_extern_bad_number(dsn, placex_table, postcode_table,
tmp_path, tokenizer): tmp_path, tokenizer):
placex_table.add(country='xx', geom='POINT(10 12)', placex_table.add(country='xx', geom='POINT(10 12)',
address=dict(postcode='AB 4511')) address=dict(postcode='AB 4511'))

12
test/python/test_tools_refresh.py
View File

@ -7,20 +7,18 @@ import pytest
from nominatim.tools import refresh from nominatim.tools import refresh
TEST_DIR = (Path(__file__) / '..' / '..').resolve()
def test_refresh_import_wikipedia_not_existing(dsn): def test_refresh_import_wikipedia_not_existing(dsn):
assert 1 == refresh.import_wikipedia_articles(dsn, Path('.')) assert refresh.import_wikipedia_articles(dsn, Path('.')) == 1
@pytest.mark.parametrize("replace", (True, False)) @pytest.mark.parametrize("replace", (True, False))
def test_refresh_import_wikipedia(dsn, table_factory, temp_db_cursor, replace): def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, replace):
if replace: if replace:
table_factory('wikipedia_article') table_factory('wikipedia_article')
table_factory('wikipedia_redirect') table_factory('wikipedia_redirect')
# use the small wikipedia file for the API testdb # use the small wikipedia file for the API testdb
assert 0 == refresh.import_wikipedia_articles(dsn, TEST_DIR / 'testdb') assert refresh.import_wikipedia_articles(dsn, src_dir / 'test' / 'testdb') == 0
assert temp_db_cursor.scalar('SELECT count(*) FROM wikipedia_article') > 0 assert temp_db_cursor.table_rows('wikipedia_article') > 0
assert temp_db_cursor.scalar('SELECT count(*) FROM wikipedia_redirect') > 0 assert temp_db_cursor.table_rows('wikipedia_redirect') > 0

12
test/python/test_tools_refresh_address_levels.py
View File

@ -11,7 +11,7 @@ from nominatim.tools.refresh import load_address_levels, load_address_levels_fro
def test_load_ranks_def_config(temp_db_conn, temp_db_cursor, def_config): def test_load_ranks_def_config(temp_db_conn, temp_db_cursor, def_config):
load_address_levels_from_file(temp_db_conn, Path(def_config.ADDRESS_LEVEL_CONFIG)) load_address_levels_from_file(temp_db_conn, Path(def_config.ADDRESS_LEVEL_CONFIG))
assert temp_db_cursor.scalar('SELECT count(*) FROM address_levels') > 0 assert temp_db_cursor.table_rows('address_levels') > 0
def test_load_ranks_from_file(temp_db_conn, temp_db_cursor, tmp_path): def test_load_ranks_from_file(temp_db_conn, temp_db_cursor, tmp_path):
test_file = tmp_path / 'test_levels.json' test_file = tmp_path / 'test_levels.json'
@ -19,7 +19,7 @@ def test_load_ranks_from_file(temp_db_conn, temp_db_cursor, tmp_path):
load_address_levels_from_file(temp_db_conn, test_file) load_address_levels_from_file(temp_db_conn, test_file)
assert temp_db_cursor.scalar('SELECT count(*) FROM address_levels') > 0 assert temp_db_cursor.table_rows('address_levels') > 0
def test_load_ranks_from_broken_file(temp_db_conn, tmp_path): def test_load_ranks_from_broken_file(temp_db_conn, tmp_path):
@ -35,7 +35,7 @@ def test_load_ranks_country(temp_db_conn, temp_db_cursor):
[{"tags": {"place": {"village": 14}}}, [{"tags": {"place": {"village": 14}}},
{"countries": ['de'], {"countries": ['de'],
"tags": {"place": {"village": 15}}}, "tags": {"place": {"village": 15}}},
{"countries": ['uk', 'us' ], {"countries": ['uk', 'us'],
"tags": {"place": {"village": 16}}} "tags": {"place": {"village": 16}}}
]) ])
@ -62,8 +62,7 @@ def test_load_ranks_default_value(temp_db_conn, temp_db_cursor):
def test_load_ranks_multiple_keys(temp_db_conn, temp_db_cursor): def test_load_ranks_multiple_keys(temp_db_conn, temp_db_cursor):
load_address_levels(temp_db_conn, 'levels', load_address_levels(temp_db_conn, 'levels',
[{"tags": [{"tags": {"place": {"city": 14},
{"place": {"city": 14},
"boundary": {"administrative2" : 4}} "boundary": {"administrative2" : 4}}
}]) }])
@ -75,8 +74,7 @@ def test_load_ranks_multiple_keys(temp_db_conn, temp_db_cursor):
def test_load_ranks_address(temp_db_conn, temp_db_cursor): def test_load_ranks_address(temp_db_conn, temp_db_cursor):
load_address_levels(temp_db_conn, 'levels', load_address_levels(temp_db_conn, 'levels',
[{"tags": [{"tags": {"place": {"city": 14,
{"place": {"city": 14,
"town" : [14, 13]}} "town" : [14, 13]}}
}]) }])

3
test/python/test_tools_refresh_create_functions.py
View File

@ -31,7 +31,8 @@ def test_create_functions(temp_db_cursor, conn, def_config, sql_tmp_path):
@pytest.mark.parametrize("dbg,ret", ((True, 43), (False, 22))) @pytest.mark.parametrize("dbg,ret", ((True, 43), (False, 22)))
def test_create_functions_with_template(temp_db_cursor, conn, def_config, sql_tmp_path, dbg, ret): def test_create_functions_with_template(temp_db_cursor, conn, def_config, sql_tmp_path,
dbg, ret):
sqlfile = sql_tmp_path / 'functions.sql' sqlfile = sql_tmp_path / 'functions.sql'
sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER
AS $$ AS $$

51
test/python/test_tools_refresh_setup_website.py
View File

@ -1,7 +1,6 @@
""" """
Tests for setting up the website scripts. Tests for setting up the website scripts.
""" """
from pathlib import Path
import subprocess import subprocess
import pytest import pytest
@ -9,67 +8,65 @@ import pytest
from nominatim.tools import refresh from nominatim.tools import refresh
@pytest.fixture @pytest.fixture
def envdir(tmpdir): def test_script(tmp_path):
(tmpdir / 'php').mkdir() (tmp_path / 'php').mkdir()
(tmpdir / 'php' / 'website').mkdir()
return tmpdir
website_dir = (tmp_path / 'php' / 'website')
website_dir.mkdir()
@pytest.fixture
def test_script(envdir):
def _create_file(code): def _create_file(code):
outfile = envdir / 'php' / 'website' / 'reverse-only-search.php' outfile = website_dir / 'reverse-only-search.php'
outfile.write_text('<?php\n{}\n'.format(code), 'utf-8') outfile.write_text('<?php\n{}\n'.format(code), 'utf-8')
return _create_file return _create_file
def run_website_script(envdir, config, conn): @pytest.fixture
config.lib_dir.php = envdir / 'php' def run_website_script(tmp_path, def_config, temp_db_conn):
config.project_dir = envdir def_config.lib_dir.php = tmp_path / 'php'
refresh.setup_website(envdir, config, conn) def_config.project_dir = tmp_path
def _runner():
refresh.setup_website(tmp_path, def_config, temp_db_conn)
proc = subprocess.run(['/usr/bin/env', 'php', '-Cq', proc = subprocess.run(['/usr/bin/env', 'php', '-Cq',
envdir / 'search.php'], check=False) tmp_path / 'search.php'], check=False)
return proc.returncode return proc.returncode
return _runner
@pytest.mark.parametrize("setting,retval", (('yes', 10), ('no', 20))) @pytest.mark.parametrize("setting,retval", (('yes', 10), ('no', 20)))
def test_setup_website_check_bool(def_config, monkeypatch, envdir, test_script, def test_setup_website_check_bool(monkeypatch, test_script, run_website_script,
setting, retval, temp_db_conn): setting, retval):
monkeypatch.setenv('NOMINATIM_CORS_NOACCESSCONTROL', setting) monkeypatch.setenv('NOMINATIM_CORS_NOACCESSCONTROL', setting)
test_script('exit(CONST_NoAccessControl ? 10 : 20);') test_script('exit(CONST_NoAccessControl ? 10 : 20);')
assert run_website_script(envdir, def_config, temp_db_conn) == retval assert run_website_script() == retval
@pytest.mark.parametrize("setting", (0, 10, 99067)) @pytest.mark.parametrize("setting", (0, 10, 99067))
def test_setup_website_check_int(def_config, monkeypatch, envdir, test_script, setting, def test_setup_website_check_int(monkeypatch, test_script, run_website_script, setting):
temp_db_conn):
monkeypatch.setenv('NOMINATIM_LOOKUP_MAX_COUNT', str(setting)) monkeypatch.setenv('NOMINATIM_LOOKUP_MAX_COUNT', str(setting))
test_script('exit(CONST_Places_Max_ID_count == {} ? 10 : 20);'.format(setting)) test_script('exit(CONST_Places_Max_ID_count == {} ? 10 : 20);'.format(setting))
assert run_website_script(envdir, def_config, temp_db_conn) == 10 assert run_website_script() == 10
def test_setup_website_check_empty_str(def_config, monkeypatch, envdir, test_script, def test_setup_website_check_empty_str(monkeypatch, test_script, run_website_script):
temp_db_conn):
monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', '') monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', '')
test_script('exit(CONST_Default_Language === false ? 10 : 20);') test_script('exit(CONST_Default_Language === false ? 10 : 20);')
assert run_website_script(envdir, def_config, temp_db_conn) == 10 assert run_website_script() == 10
def test_setup_website_check_str(def_config, monkeypatch, envdir, test_script, def test_setup_website_check_str(monkeypatch, test_script, run_website_script):
temp_db_conn):
monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'ffde 2') monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'ffde 2')
test_script('exit(CONST_Default_Language === "ffde 2" ? 10 : 20);') test_script('exit(CONST_Default_Language === "ffde 2" ? 10 : 20);')
assert run_website_script(envdir, def_config, temp_db_conn) == 10 assert run_website_script() == 10

39
test/python/test_tools_replication.py
View File

@ -18,48 +18,51 @@ OSM_NODE_DATA = """\
</osm> </osm>
""" """
@pytest.fixture(autouse=True)
def setup_status_table(status_table):
pass
### init replication ### init replication
def test_init_replication_bad_base_url(monkeypatch, status_table, place_row, temp_db_conn, temp_db_cursor): def test_init_replication_bad_base_url(monkeypatch, place_row, temp_db_conn):
place_row(osm_type='N', osm_id=100) place_row(osm_type='N', osm_id=100)
monkeypatch.setattr(nominatim.db.status, "get_url", lambda u : OSM_NODE_DATA) monkeypatch.setattr(nominatim.db.status, "get_url", lambda u: OSM_NODE_DATA)
with pytest.raises(UsageError, match="Failed to reach replication service"): with pytest.raises(UsageError, match="Failed to reach replication service"):
nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io') nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io')
def test_init_replication_success(monkeypatch, status_table, place_row, temp_db_conn, temp_db_cursor): def test_init_replication_success(monkeypatch, place_row, temp_db_conn, temp_db_cursor):
place_row(osm_type='N', osm_id=100) place_row(osm_type='N', osm_id=100)
monkeypatch.setattr(nominatim.db.status, "get_url", lambda u : OSM_NODE_DATA) monkeypatch.setattr(nominatim.db.status, "get_url", lambda u: OSM_NODE_DATA)
monkeypatch.setattr(nominatim.tools.replication.ReplicationServer, monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
"timestamp_to_sequence", "timestamp_to_sequence",
lambda self, date: 234) lambda self, date: 234)
nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io') nominatim.tools.replication.init_replication(temp_db_conn, 'https://test.io')
temp_db_cursor.execute("SELECT * FROM import_status")
expected_date = dt.datetime.strptime('2006-01-27T19:09:10', status.ISODATE_FORMAT)\ expected_date = dt.datetime.strptime('2006-01-27T19:09:10', status.ISODATE_FORMAT)\
.replace(tzinfo=dt.timezone.utc) .replace(tzinfo=dt.timezone.utc)
assert temp_db_cursor.rowcount == 1
assert temp_db_cursor.fetchone() == [expected_date, 234, True] assert temp_db_cursor.row_set("SELECT * FROM import_status") \
== {(expected_date, 234, True)}
### checking for updates ### checking for updates
def test_check_for_updates_empty_status_table(status_table, temp_db_conn): def test_check_for_updates_empty_status_table(temp_db_conn):
assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254 assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
def test_check_for_updates_seq_not_set(status_table, temp_db_conn): def test_check_for_updates_seq_not_set(temp_db_conn):
status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc)) status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc))
assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254 assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
def test_check_for_updates_no_state(monkeypatch, status_table, temp_db_conn): def test_check_for_updates_no_state(monkeypatch, temp_db_conn):
status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=345) status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=345)
monkeypatch.setattr(nominatim.tools.replication.ReplicationServer, monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
@ -69,7 +72,7 @@ def test_check_for_updates_no_state(monkeypatch, status_table, temp_db_conn):
@pytest.mark.parametrize("server_sequence,result", [(344, 2), (345, 2), (346, 0)]) @pytest.mark.parametrize("server_sequence,result", [(344, 2), (345, 2), (346, 0)])
def test_check_for_updates_no_new_data(monkeypatch, status_table, temp_db_conn, def test_check_for_updates_no_new_data(monkeypatch, temp_db_conn,
server_sequence, result): server_sequence, result):
date = dt.datetime.now(dt.timezone.utc) date = dt.datetime.now(dt.timezone.utc)
status.set_status(temp_db_conn, date, seq=345) status.set_status(temp_db_conn, date, seq=345)
@ -91,19 +94,19 @@ def update_options(tmpdir):
import_file=tmpdir / 'foo.osm', import_file=tmpdir / 'foo.osm',
max_diff_size=1) max_diff_size=1)
def test_update_empty_status_table(status_table, temp_db_conn): def test_update_empty_status_table(temp_db_conn):
with pytest.raises(UsageError): with pytest.raises(UsageError):
nominatim.tools.replication.update(temp_db_conn, {}) nominatim.tools.replication.update(temp_db_conn, {})
def test_update_already_indexed(status_table, temp_db_conn): def test_update_already_indexed(temp_db_conn):
status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False) status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \ assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \
== nominatim.tools.replication.UpdateState.MORE_PENDING == nominatim.tools.replication.UpdateState.MORE_PENDING
def test_update_no_data_no_sleep(monkeypatch, status_table, temp_db_conn, update_options): def test_update_no_data_no_sleep(monkeypatch, temp_db_conn, update_options):
date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1) date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
status.set_status(temp_db_conn, date, seq=34) status.set_status(temp_db_conn, date, seq=34)
@ -112,7 +115,7 @@ def test_update_no_data_no_sleep(monkeypatch, status_table, temp_db_conn, update
lambda *args, **kwargs: None) lambda *args, **kwargs: None)
sleeptime = [] sleeptime = []
monkeypatch.setattr(time, 'sleep', lambda s: sleeptime.append(s)) monkeypatch.setattr(time, 'sleep', sleeptime.append)
assert nominatim.tools.replication.update(temp_db_conn, update_options) \ assert nominatim.tools.replication.update(temp_db_conn, update_options) \
== nominatim.tools.replication.UpdateState.NO_CHANGES == nominatim.tools.replication.UpdateState.NO_CHANGES
@ -120,7 +123,7 @@ def test_update_no_data_no_sleep(monkeypatch, status_table, temp_db_conn, update
assert not sleeptime assert not sleeptime
def test_update_no_data_sleep(monkeypatch, status_table, temp_db_conn, update_options): def test_update_no_data_sleep(monkeypatch, temp_db_conn, update_options):
date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30) date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
status.set_status(temp_db_conn, date, seq=34) status.set_status(temp_db_conn, date, seq=34)
@ -129,7 +132,7 @@ def test_update_no_data_sleep(monkeypatch, status_table, temp_db_conn, update_op
lambda *args, **kwargs: None) lambda *args, **kwargs: None)
sleeptime = [] sleeptime = []
monkeypatch.setattr(time, 'sleep', lambda s: sleeptime.append(s)) monkeypatch.setattr(time, 'sleep', sleeptime.append)
assert nominatim.tools.replication.update(temp_db_conn, update_options) \ assert nominatim.tools.replication.update(temp_db_conn, update_options) \
== nominatim.tools.replication.UpdateState.NO_CHANGES == nominatim.tools.replication.UpdateState.NO_CHANGES

18
test/python/test_tools_sp_csv_loader.py
View File

@ -1,12 +1,10 @@
""" """
Tests for methods of the SPCsvLoader class. Tests for methods of the SPCsvLoader class.
""" """
from nominatim.errors import UsageError
import pytest import pytest
from pathlib import Path
from nominatim.tools.special_phrases.sp_csv_loader import SPCsvLoader
TEST_BASE_DIR = Path(__file__) / '..' / '..' from nominatim.errors import UsageError
from nominatim.tools.special_phrases.sp_csv_loader import SPCsvLoader
def test_parse_csv(sp_csv_loader): def test_parse_csv(sp_csv_loader):
""" """
@ -43,16 +41,20 @@ def check_phrases_content(phrases):
the right phrases of the sp_csv_test.csv special phrases. the right phrases of the sp_csv_test.csv special phrases.
""" """
return len(phrases) > 1 \ return len(phrases) > 1 \
and any(p.p_label == 'Billboard' and p.p_class == 'advertising' and p.p_type == 'billboard' and any(p.p_label == 'Billboard'
and p.p_class == 'advertising'
and p.p_type == 'billboard'
and p.p_operator == '-' for p in phrases) \ and p.p_operator == '-' for p in phrases) \
and any(p.p_label == 'Zip Lines' and p.p_class == 'aerialway' and p.p_type == 'zip_line' and any(p.p_label == 'Zip Lines'
and p.p_class == 'aerialway'
and p.p_type == 'zip_line'
and p.p_operator == '-' for p in phrases) and p.p_operator == '-' for p in phrases)
@pytest.fixture @pytest.fixture
def sp_csv_loader(): def sp_csv_loader(src_dir):
""" """
Return an instance of SPCsvLoader. Return an instance of SPCsvLoader.
""" """
csv_path = (TEST_BASE_DIR / 'testdata' / 'sp_csv_test.csv').resolve() csv_path = (src_dir / 'test' / 'testdata' / 'sp_csv_test.csv').resolve()
loader = SPCsvLoader(csv_path) loader = SPCsvLoader(csv_path)
return loader return loader

51
test/python/test_tools_sp_wiki_loader.py
View File

@ -2,18 +2,34 @@
Tests for methods of the SPWikiLoader class. Tests for methods of the SPWikiLoader class.
""" """
import pytest import pytest
from pathlib import Path
from nominatim.tools.special_phrases.sp_wiki_loader import SPWikiLoader from nominatim.tools.special_phrases.sp_wiki_loader import SPWikiLoader
TEST_BASE_DIR = Path(__file__) / '..' / '..' @pytest.fixture
def xml_wiki_content(src_dir):
"""
return the content of the static xml test file.
"""
xml_test_content = src_dir / 'test' / 'testdata' / 'special_phrases_test_content.txt'
return xml_test_content.read_text()
def test_parse_xml(sp_wiki_loader):
@pytest.fixture
def sp_wiki_loader(monkeypatch, def_config, xml_wiki_content):
"""
Return an instance of SPWikiLoader.
"""
loader = SPWikiLoader(def_config, ['en'])
monkeypatch.setattr('nominatim.tools.special_phrases.sp_wiki_loader.SPWikiLoader._get_wiki_content',
lambda self, lang: xml_wiki_content)
return loader
def test_parse_xml(sp_wiki_loader, xml_wiki_content):
""" """
Test method parse_xml() Test method parse_xml()
Should return the right SpecialPhrase objects. Should return the right SpecialPhrase objects.
""" """
xml = get_test_xml_wiki_content() phrases = sp_wiki_loader.parse_xml(xml_wiki_content)
phrases = sp_wiki_loader.parse_xml(xml)
assert check_phrases_content(phrases) assert check_phrases_content(phrases)
@ -36,28 +52,3 @@ def check_phrases_content(phrases):
and p.p_operator == '-' for p in phrases) \ and p.p_operator == '-' for p in phrases) \
and any(p.p_label == 'Zip Line' and p.p_class == 'aerialway' and p.p_type == 'zip_line' and any(p.p_label == 'Zip Line' and p.p_class == 'aerialway' and p.p_type == 'zip_line'
and p.p_operator == '-' for p in phrases) and p.p_operator == '-' for p in phrases)
@pytest.fixture
def sp_wiki_loader(monkeypatch, def_config):
"""
Return an instance of SPWikiLoader.
"""
loader = SPWikiLoader(def_config, ['en'])
monkeypatch.setattr('nominatim.tools.special_phrases.sp_wiki_loader.SPWikiLoader._get_wiki_content',
mock_get_wiki_content)
return loader
def mock_get_wiki_content(self, lang):
"""
Mock the _get_wiki_content() method to return
static xml test file content.
"""
return get_test_xml_wiki_content()
def get_test_xml_wiki_content():
"""
return the content of the static xml test file.
"""
xml_test_content_path = (TEST_BASE_DIR / 'testdata' / 'special_phrases_test_content.txt').resolve()
with open(xml_test_content_path) as xml_content_reader:
return xml_content_reader.read()

13
test/python/test_tools_tiger_data.py
View File

@ -1,13 +1,12 @@
""" """
Test for tiger data function Test for tiger data function
""" """
from pathlib import Path import tarfile
from textwrap import dedent from textwrap import dedent
import pytest import pytest
import tarfile
from nominatim.tools import tiger_data, database_import from nominatim.tools import tiger_data
from nominatim.errors import UsageError from nominatim.errors import UsageError
class MockTigerTable: class MockTigerTable:
@ -42,7 +41,8 @@ def tiger_table(def_config, temp_db_conn, sql_preprocessor,
stop INTEGER, interpol TEXT, stop INTEGER, interpol TEXT,
token_info JSONB, postcode TEXT) token_info JSONB, postcode TEXT)
RETURNS INTEGER AS $$ RETURNS INTEGER AS $$
INSERT INTO tiger VALUES(linegeo, start, stop, interpol, token_info, postcode) RETURNING 1 INSERT INTO tiger VALUES(linegeo, start, stop, interpol, token_info, postcode)
RETURNING 1
$$ LANGUAGE SQL;""") $$ LANGUAGE SQL;""")
(def_config.lib_dir.sql / 'tiger_import_finish.sql').write_text( (def_config.lib_dir.sql / 'tiger_import_finish.sql').write_text(
"""DROP FUNCTION tiger_line_import (linegeo GEOMETRY, in_startnumber INTEGER, """DROP FUNCTION tiger_line_import (linegeo GEOMETRY, in_startnumber INTEGER,
@ -110,7 +110,7 @@ def test_add_tiger_data_tarfile(def_config, tiger_table, tokenizer_mock,
tar.add(str(src_dir / 'test' / 'testdb' / 'tiger' / '01001.csv')) tar.add(str(src_dir / 'test' / 'testdb' / 'tiger' / '01001.csv'))
tar.close() tar.close()
tiger_data.add_tiger_data(str(tmp_path / 'sample.tar.gz'), def_config, 1, tiger_data.add_tiger_data(str(tmp_path / 'sample.tar.gz'), def_config, threads,
tokenizer_mock()) tokenizer_mock())
assert tiger_table.count() == 6213 assert tiger_table.count() == 6213
@ -126,7 +126,7 @@ def test_add_tiger_data_bad_tarfile(def_config, tiger_table, tokenizer_mock,
def test_add_tiger_data_empty_tarfile(def_config, tiger_table, tokenizer_mock, def test_add_tiger_data_empty_tarfile(def_config, tiger_table, tokenizer_mock,
tmp_path, src_dir): tmp_path):
tar = tarfile.open(str(tmp_path / 'sample.tar.gz'), "w:gz") tar = tarfile.open(str(tmp_path / 'sample.tar.gz'), "w:gz")
tar.add(__file__) tar.add(__file__)
tar.close() tar.close()
@ -135,4 +135,3 @@ def test_add_tiger_data_empty_tarfile(def_config, tiger_table, tokenizer_mock,
tokenizer_mock()) tokenizer_mock())
assert tiger_table.count() == 0 assert tiger_table.count() == 0