osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-12-26 06:22:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added fixture for sql_preprocessor and fixed some issues

Browse Source
This commit is contained in:
Darkshredder 2021-03-11 15:39:17 +05:30
parent 8486a83cf5
commit e5719de657
4 changed files with 12 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nominatim/tools/tiger_data.py
View File

@ -44,9 +44,7 @@ def handle_threaded_sql_statements(sel, file):
lines = 0
end_of_file = False
# Using pool of database connections to execute sql statements
while True:
if end_of_file:
break
while not end_of_file:
for key, _ in sel.select(1):
conn = key.data
try:
@ -61,7 +59,7 @@ def handle_threaded_sql_statements(sel, file):
print('. ', end='', flush=True)
lines = 0
except Exception as exc: # pylint: disable=broad-except
LOG.error('Wrong SQL statement: %s', exc)
LOG.info('Wrong SQL statement: %s', exc)
def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):

7
test/python/conftest.py
View File

@ -13,6 +13,7 @@ sys.path.insert(0, str(SRC_DIR.resolve()))
from nominatim.config import Configuration
from nominatim.db import connection
from nominatim.db.sql_preprocessor import SQLPreprocessor
class _TestingCursor(psycopg2.extras.DictCursor):
""" Extension to the DictCursor class that provides execution
@ -266,3 +267,9 @@ def osm2pgsql_options(temp_db):
flatnode_file='',
tablespaces=dict(slim_data='', slim_index='',
main_data='', main_index=''))
@pytest.fixture
def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory):
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
table_factory('country_name', 'partition INT', (0, 1, 2))
return SQLPreprocessor(temp_db_conn, def_config, tmp_path)

9
test/python/test_db_sql_preprocessor.py
View File

@ -5,8 +5,6 @@ from pathlib import Path
import pytest
from nominatim.db.sql_preprocessor import SQLPreprocessor
@pytest.fixture
def sql_factory(tmp_path):
def _mk_sql(sql_body):
@ -21,13 +19,6 @@ def sql_factory(tmp_path):
return _mk_sql
@pytest.fixture
def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory):
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
table_factory('country_name', 'partition INT', (0, 1, 2))
return SQLPreprocessor(temp_db_conn, def_config, tmp_path)
@pytest.mark.parametrize("expr,ret", [
("'a'", 'a'),
("'{{db.partitions|join}}'", '012'),

14
test/python/test_tools_tiger_data.py
View File

@ -10,15 +10,11 @@ from nominatim.tools import tiger_data, database_import
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data(dsn, src_dir, def_config, monkeypatch,tmp_path,
def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor,
temp_db_cursor, threads, temp_db):
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)')
database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
ignore_partitions=False)
sqlfile = tmp_path / '1010.sql'
sqlfile.write_text("""INSERT INTO place values (1)""")
tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
@ -26,15 +22,11 @@ def test_add_tiger_data(dsn, src_dir, def_config, monkeypatch,tmp_path,
assert temp_db_cursor.table_rows('place') == 1
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_tarfile(dsn, src_dir, def_config, monkeypatch,tmp_path,
temp_db_cursor, threads, temp_db):
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
temp_db_cursor, threads, temp_db, sql_preprocessor):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
temp_db_cursor.execute('CREATE TABLE place (id INT)')
database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
ignore_partitions=False)
sqlfile = tmp_path / '1010.sql'
sqlfile.write_text("""INSERT INTO place values (1)""")
tar = tarfile.open("sample.tar.gz", "w:gz")