Merge pull request #3050 from mtmail/tiger-check-if-database-frozen

when adding Tiger data, check first if database is in frozen state

.github/actions/build-nominatim/action.yml

@@ -29,8 +29,8 @@ runs:
             if [ "$FLAVOUR" == "oldstuff" ]; then
                 pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4 GeoAlchemy2==0.10.0 datrie asyncpg
             else
-                sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml python3-asyncpg
-                pip3 install sqlalchemy GeoAlchemy2
+                sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
+                pip3 install sqlalchemy GeoAlchemy2 asyncpg
             fi
           shell: bash
           env:

nominatim/tools/freeze.py

@@ -50,3 +50,9 @@ def drop_flatnode_file(fpath: Optional[Path]) -> None:
     """
     if fpath and fpath.exists():
         fpath.unlink()
+
+def is_frozen(conn: Connection) -> bool:
+    """ Returns true if database is in a frozen state
+    """
+
+    return conn.table_exists('place') is False

nominatim/tools/tiger_data.py

@@ -23,6 +23,7 @@ from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.errors import UsageError
 from nominatim.data.place_info import PlaceInfo
 from nominatim.tokenizer.base import AbstractAnalyzer, AbstractTokenizer
+from nominatim.tools import freeze
 
 LOG = logging.getLogger()
 
@@ -113,6 +114,13 @@ def add_tiger_data(data_dir: str, config: Configuration, threads: int,
     """
     dsn = config.get_libpq_dsn()
 
+    with connect(dsn) as conn:
+        is_frozen = freeze.is_frozen(conn)
+        conn.close()
+
+        if is_frozen:
+            raise UsageError("Tiger cannot be imported when database frozen (Github issue #3048)")
+
     with TigerInput(data_dir) as tar:
         if not tar:
             return 1

test/python/tools/test_freeze.py

@@ -30,6 +30,8 @@ def test_drop_tables(temp_db_conn, temp_db_cursor, table_factory):
     for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES:
         table_factory(table)
 
+    assert not freeze.is_frozen(temp_db_conn)
+
     freeze.drop_update_tables(temp_db_conn)
 
     for table in NOMINATIM_RUNTIME_TABLES:
@@ -38,6 +40,8 @@ def test_drop_tables(temp_db_conn, temp_db_cursor, table_factory):
     for table in NOMINATIM_DROP_TABLES:
         assert not temp_db_cursor.table_exists(table)
 
+    assert freeze.is_frozen(temp_db_conn)
+
 def test_drop_flatnode_file_no_file():
     freeze.drop_flatnode_file(None)
 
@@ -46,7 +50,7 @@ def test_drop_flatnode_file_file_already_gone(tmp_path):
     freeze.drop_flatnode_file(tmp_path / 'something.store')
 
 
-def test_drop_flatnode_file_delte(tmp_path):
+def test_drop_flatnode_file_delete(tmp_path):
     flatfile = tmp_path / 'flatnode.store'
     flatfile.write_text('Some content')
 

test/python/tools/test_tiger_data.py

@@ -12,7 +12,7 @@ from textwrap import dedent
 
 import pytest
 
-from nominatim.tools import tiger_data
+from nominatim.tools import tiger_data, freeze
 from nominatim.errors import UsageError
 
 class MockTigerTable:
@@ -27,6 +27,9 @@ class MockTigerTable:
                                                token_info JSONB,
                                                postcode TEXT)""")
 
+            # We need this table to determine if the database is frozen or not
+            cur.execute("CREATE TABLE place (number INTEGER)")
+
     def count(self):
         with self.conn.cursor() as cur:
             return cur.scalar("SELECT count(*) FROM tiger")
@@ -80,6 +83,17 @@ def test_add_tiger_data(def_config, src_dir, tiger_table, tokenizer_mock, thread
     assert tiger_table.count() == 6213
 
 
+def test_add_tiger_data_database_frozen(def_config, temp_db_conn, tiger_table, tokenizer_mock,
+                                 tmp_path):
+    freeze.drop_update_tables(temp_db_conn)
+
+    with pytest.raises(UsageError) as excinfo:
+        tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())
+
+        assert "database frozen" in str(excinfo.value)
+
+    assert tiger_table.count() == 0
+
 def test_add_tiger_data_no_files(def_config, tiger_table, tokenizer_mock,
                                  tmp_path):
     tiger_data.add_tiger_data(str(tmp_path), def_config, 1, tokenizer_mock())