From e5719de65767085202afeaa68a01464d82b11eb6 Mon Sep 17 00:00:00 2001 From: Darkshredder Date: Thu, 11 Mar 2021 15:39:17 +0530 Subject: [PATCH] Added fixture for sql_preprocessor and fixed some issues --- nominatim/tools/tiger_data.py | 6 ++---- test/python/conftest.py | 7 +++++++ test/python/test_db_sql_preprocessor.py | 9 --------- test/python/test_tools_tiger_data.py | 14 +++----------- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py index e3adfd33..9b960e2d 100644 --- a/nominatim/tools/tiger_data.py +++ b/nominatim/tools/tiger_data.py @@ -44,9 +44,7 @@ def handle_threaded_sql_statements(sel, file): lines = 0 end_of_file = False # Using pool of database connections to execute sql statements - while True: - if end_of_file: - break + while not end_of_file: for key, _ in sel.select(1): conn = key.data try: @@ -61,7 +59,7 @@ def handle_threaded_sql_statements(sel, file): print('. ', end='', flush=True) lines = 0 except Exception as exc: # pylint: disable=broad-except - LOG.error('Wrong SQL statement: %s', exc) + LOG.info('Wrong SQL statement: %s', exc) def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir): diff --git a/test/python/conftest.py b/test/python/conftest.py index d16dceff..4b7cccc3 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -13,6 +13,7 @@ sys.path.insert(0, str(SRC_DIR.resolve())) from nominatim.config import Configuration from nominatim.db import connection +from nominatim.db.sql_preprocessor import SQLPreprocessor class _TestingCursor(psycopg2.extras.DictCursor): """ Extension to the DictCursor class that provides execution @@ -266,3 +267,9 @@ def osm2pgsql_options(temp_db): flatnode_file='', tablespaces=dict(slim_data='', slim_index='', main_data='', main_index='')) + +@pytest.fixture +def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory): + monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.') + table_factory('country_name', 'partition INT', (0, 1, 2)) + return SQLPreprocessor(temp_db_conn, def_config, tmp_path) diff --git a/test/python/test_db_sql_preprocessor.py b/test/python/test_db_sql_preprocessor.py index 3c10000f..08a195bd 100644 --- a/test/python/test_db_sql_preprocessor.py +++ b/test/python/test_db_sql_preprocessor.py @@ -5,8 +5,6 @@ from pathlib import Path import pytest -from nominatim.db.sql_preprocessor import SQLPreprocessor - @pytest.fixture def sql_factory(tmp_path): def _mk_sql(sql_body): @@ -21,13 +19,6 @@ def sql_factory(tmp_path): return _mk_sql - -@pytest.fixture -def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory): - monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.') - table_factory('country_name', 'partition INT', (0, 1, 2)) - return SQLPreprocessor(temp_db_conn, def_config, tmp_path) - @pytest.mark.parametrize("expr,ret", [ ("'a'", 'a'), ("'{{db.partitions|join}}'", '012'), diff --git a/test/python/test_tools_tiger_data.py b/test/python/test_tools_tiger_data.py index 6290b994..5029a132 100644 --- a/test/python/test_tools_tiger_data.py +++ b/test/python/test_tools_tiger_data.py @@ -10,15 +10,11 @@ from nominatim.tools import tiger_data, database_import @pytest.mark.parametrize("threads", (1, 5)) -def test_add_tiger_data(dsn, src_dir, def_config, monkeypatch,tmp_path, +def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor, temp_db_cursor, threads, temp_db): - monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.') temp_db_cursor.execute('CREATE EXTENSION hstore') temp_db_cursor.execute('CREATE EXTENSION postgis') temp_db_cursor.execute('CREATE TABLE place (id INT)') - - database_import.import_base_data('dbname=' + temp_db, src_dir / 'data', - ignore_partitions=False) sqlfile = tmp_path / '1010.sql' sqlfile.write_text("""INSERT INTO place values (1)""") tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql') @@ -26,15 +22,11 @@ def test_add_tiger_data(dsn, src_dir, def_config, monkeypatch,tmp_path, assert temp_db_cursor.table_rows('place') == 1 @pytest.mark.parametrize("threads", (1, 5)) -def test_add_tiger_data_tarfile(dsn, src_dir, def_config, monkeypatch,tmp_path, - temp_db_cursor, threads, temp_db): - monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.') +def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path, + temp_db_cursor, threads, temp_db, sql_preprocessor): temp_db_cursor.execute('CREATE EXTENSION hstore') temp_db_cursor.execute('CREATE EXTENSION postgis') temp_db_cursor.execute('CREATE TABLE place (id INT)') - - database_import.import_base_data('dbname=' + temp_db, src_dir / 'data', - ignore_partitions=False) sqlfile = tmp_path / '1010.sql' sqlfile.write_text("""INSERT INTO place values (1)""") tar = tarfile.open("sample.tar.gz", "w:gz")