From e5f332bd71f626a146dac7dcaeda83f1aec53a46 Mon Sep 17 00:00:00 2001 From: Marc Tobias Date: Tue, 2 May 2023 18:37:36 +0200 Subject: [PATCH] when adding Tiger data, check first if database is in frozen state --- .github/actions/build-nominatim/action.yml | 4 ++-- nominatim/tools/freeze.py | 6 ++++++ nominatim/tools/tiger_data.py | 8 ++++++++ test/python/tools/test_freeze.py | 6 +++++- test/python/tools/test_tiger_data.py | 16 +++++++++++++++- 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index d5f04efe..6508f03e 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -29,8 +29,8 @@ runs: if [ "$FLAVOUR" == "oldstuff" ]; then pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 GeoAlchemy2==0.10.0 datrie asyncpg else - sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml python3-asyncpg - pip3 install sqlalchemy GeoAlchemy2 + sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml + pip3 install sqlalchemy GeoAlchemy2 asyncpg fi shell: bash env: diff --git a/nominatim/tools/freeze.py b/nominatim/tools/freeze.py index 39c3279d..602def55 100644 --- a/nominatim/tools/freeze.py +++ b/nominatim/tools/freeze.py @@ -50,3 +50,9 @@ def drop_flatnode_file(fpath: Optional[Path]) -> None: """ if fpath and fpath.exists(): fpath.unlink() + +def is_frozen(conn: Connection) -> bool: + """ Returns true if database is in a frozen state + """ + + return conn.table_exists('place') is False diff --git a/nominatim/tools/tiger_data.py b/nominatim/tools/tiger_data.py index 4a32bb1e..70cecae5 100644 --- a/nominatim/tools/tiger_data.py +++ b/nominatim/tools/tiger_data.py @@ -23,6 +23,7 @@ from nominatim.db.sql_preprocessor import SQLPreprocessor from nominatim.errors import UsageError from nominatim.data.place_info import PlaceInfo from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer +from nominatim.tools import freeze LOG = logging.getLogger() @@ -113,6 +114,13 @@ def add_tiger_data(data_dir: str, config: Configuration, threads: int, """ dsn = config.get_libpq_dsn() + with connect(dsn) as conn: + is_frozen = freeze.is_frozen(conn) + conn.close() + + if is_frozen: + raise UsageError("Tiger cannot be imported when database frozen (Github issue #3048)") + with TigerInput(data_dir) as tar: if not tar: return 1 diff --git a/test/python/tools/test_freeze.py b/test/python/tools/test_freeze.py index 3ebb1730..0d44501a 100644 --- a/test/python/tools/test_freeze.py +++ b/test/python/tools/test_freeze.py @@ -30,6 +30,8 @@ def test_drop_tables(temp_db_conn, temp_db_cursor, table_factory): for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES: table_factory(table) + assert not freeze.is_frozen(temp_db_conn) + freeze.drop_update_tables(temp_db_conn) for table in NOMINATIM_RUNTIME_TABLES: @@ -38,6 +40,8 @@ def test_drop_tables(temp_db_conn, temp_db_cursor, table_factory): for table in NOMINATIM_DROP_TABLES: assert not temp_db_cursor.table_exists(table) + assert freeze.is_frozen(temp_db_conn) + def test_drop_flatnode_file_no_file(): freeze.drop_flatnode_file(None) @@ -46,7 +50,7 @@ def test_drop_flatnode_file_file_already_gone(tmp_path): freeze.drop_flatnode_file(tmp_path / 'something.store') -def test_drop_flatnode_file_delte(tmp_path): +def test_drop_flatnode_file_delete(tmp_path): flatfile = tmp_path / 'flatnode.store' flatfile.write_text('Some content') diff --git a/test/python/tools/test_tiger_data.py b/test/python/tools/test_tiger_data.py index f027c4ff..4b1a1b89 100644 --- a/test/python/tools/test_tiger_data.py +++ b/test/python/tools/test_tiger_data.py @@ -12,7 +12,7 @@ from textwrap import dedent import pytest -from nominatim.tools import tiger_data +from nominatim.tools import tiger_data, freeze from nominatim.errors import UsageError class MockTigerTable: @@ -27,6 +27,9 @@ class MockTigerTable: token_info JSONB, postcode TEXT)""") + # We need this table to determine if the database is frozen or not + cur.execute("CREATE TABLE place (number INTEGER)") + def count(self): with self.conn.cursor() as cur: return cur.scalar("SELECT count(*) FROM tiger") @@ -80,6 +83,17 @@ def test_add_tiger_data(def_config, src_dir, tiger_table, tokenizer_mock, thread assert tiger_table.count() == 6213 +def test_add_tiger_data_database_frozen(def_config, temp_db_conn, tiger_table, tokenizer_mock, + tmp_path): + freeze.drop_update_tables(temp_db_conn) + + with pytest.raises(UsageError) as excinfo: + tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock()) + + assert "database frozen" in str(excinfo.value) + + assert tiger_table.count() == 0 + def test_add_tiger_data_no_files(def_config, tiger_table, tokenizer_mock, tmp_path): tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())