port check-for-update function to python

lib/admin/update.php

@@ -126,18 +126,7 @@ if ($aResult['init-updates']) {
 }
 
 if ($aResult['check-for-updates']) {
-    $aLastState = $oDB->getRow('SELECT sequence_id FROM import_status');
+    exit((clone($oNominatimCmd))->addParams('replication', '--check-for-updates')->run());
-
-    if (!$aLastState['sequence_id']) {
-        fail('Updates not set up. Please run ./utils/update.php --init-updates.');
-    }
-
-    $oCmd = (new \Nominatim\Shell(CONST_BinDir.'/check_server_for_updates.py'))
-            ->addParams($sBaseURL)
-            ->addParams($aLastState['sequence_id']);
-    $iRet = $oCmd->run();
-
-    exit($iRet);
 }
 
 if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {

nominatim/cli.py

@@ -257,14 +257,16 @@ class UpdateReplication:
             return 0
 
         if args.check_for_updates:
-            params.append('--check-for-updates')
+            ret = replication.check_for_updates(conn, args.config.REPLICATION_URL)
+            conn.close()
+            return ret
+
+        if args.once:
+            params.append('--import-osmosis')
         else:
-            if args.once:
+            params.append('--import-osmosis-all')
-                params.append('--import-osmosis')
+        if not args.do_index:
-            else:
+            params.append('--no-index')
-                params.append('--import-osmosis-all')
-            if not args.do_index:
-                params.append('--no-index')
 
         return run_legacy_script(*params, nominatim_env=args)
 

nominatim/db/status.py

@@ -48,3 +48,16 @@ def set_status(conn, date, seq=None, indexed=True):
                        VALUES (%s, %s, %s)""", (date, seq, indexed))
 
     conn.commit()
+
+
+def get_status(conn):
+    """ Return the current status as a triple of (date, sequence, indexed).
+        If status has not been set up yet, a triple of None is returned.
+    """
+    with conn.cursor() as cur:
+        cur.execute("SELECT * FROM import_status LIMIT 1")
+        if cur.rowcount < 1:
+            return None, None, None
+
+        row = cur.fetchone()
+        return row['lastimportdate'], row['sequence_id'], row['indexed']

nominatim/tools/replication.py

@@ -32,3 +32,28 @@ def init_replication(conn, base_url):
     status.set_status(conn, date=date, seq=seq)
 
     LOG.warning("Updates intialised at sequence %s (%s)", seq, date)
+
+
+def check_for_updates(conn, base_url):
+    """ Check if new data is available from the replication service at the
+        given base URL.
+    """
+    _, seq, _ = status.get_status(conn)
+
+    if seq is None:
+        LOG.error("Replication not set up. "
+                  "Please run 'nominatim replication --init' first.")
+        return 254
+
+    state = ReplicationServer(base_url).get_state_info()
+
+    if state is None:
+        LOG.error("Cannot get state for URL %s.", base_url)
+        return 253
+
+    if state.sequence <= seq:
+        LOG.warning("Database is up to date.")
+        return 1
+
+    LOG.warning("New data available (%i => %i).", seq, state.sequence)
+    return 0

test/python/test_cli.py

@@ -120,7 +120,6 @@ def test_refresh_command(monkeypatch, temp_db, command, func):
     monkeypatch.setattr(nominatim.tools.refresh, func, func_mock)
 
     assert 0 == call_nominatim('refresh', '--' + command)
-
     assert func_mock.called == 1
 
 
@@ -132,14 +131,14 @@ def test_refresh_importance_computed_after_wiki_import(mock_run_legacy, temp_db)
 
 
 @pytest.mark.parametrize("params,func", [
-                         (('--init', '--no-update-functions'), 'init_replication')
+                         (('--init', '--no-update-functions'), 'init_replication'),
+                         (('--check-for-updates',), 'check_for_updates')
                          ])
 def test_replication_command(monkeypatch, temp_db, params, func):
     func_mock = MockParamCapture()
     monkeypatch.setattr(nominatim.tools.replication, func, func_mock)
 
     assert 0 == call_nominatim('replication', *params)
-
     assert func_mock.called == 1
 
 

test/python/test_db_status.py

@@ -72,3 +72,15 @@ def test_set_status_filled_table(status_table, temp_db_conn, temp_db_cursor):
 
     assert temp_db_cursor.rowcount == 1
     assert temp_db_cursor.fetchone() == [date, 456, False]
+
+
+def test_get_status_empty_table(status_table, temp_db_conn):
+    assert nominatim.db.status.get_status(temp_db_conn) == (None, None, None)
+
+
+def test_get_status_success(status_table, temp_db_conn):
+    date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
+    nominatim.db.status.set_status(temp_db_conn, date=date, seq=667, indexed=False)
+
+    assert nominatim.db.status.get_status(temp_db_conn) == \
+             (date, 667, False)

test/python/test_tools_replication.py

@@ -4,8 +4,10 @@ Tests for replication functionality.
 import datetime as dt
 
 import pytest
+from osmium.replication.server import OsmosisState
 
 import nominatim.tools.replication
+import nominatim.db.status as status
 
 OSM_NODE_DATA = """\
 <osm version="0.6" generator="OpenStreetMap server" copyright="OpenStreetMap and contributors" attribution="http://www.openstreetmap.org/copyright" license="http://opendatacommons.org/licenses/odbl/1-0/">
@@ -39,3 +41,37 @@ def test_init_replication_success(monkeypatch, status_table, place_row, temp_db_
     expected_date = dt.datetime.fromisoformat('2006-01-27T19:09:10').replace(tzinfo=dt.timezone.utc)
     assert temp_db_cursor.rowcount == 1
     assert temp_db_cursor.fetchone() == [expected_date, 234, True]
+
+
+def test_check_for_updates_empty_status_table(status_table, temp_db_conn):
+    assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
+
+
+def test_check_for_updates_seq_not_set(status_table, temp_db_conn):
+    status.set_status(temp_db_conn, dt.datetime.now().replace(tzinfo=dt.timezone.utc))
+
+    assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
+
+
+def test_check_for_updates_no_state(monkeypatch, status_table, temp_db_conn):
+    status.set_status(temp_db_conn,
+                      dt.datetime.now().replace(tzinfo=dt.timezone.utc),
+                      seq=345)
+
+    monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
+                        "get_state_info", lambda self: None)
+
+    assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 253
+
+
+@pytest.mark.parametrize("server_sequence,result", [(344, 1), (345, 1), (346, 0)])
+def test_check_for_updates_no_new_data(monkeypatch, status_table, temp_db_conn,
+                                       server_sequence, result):
+    date = dt.datetime.now().replace(tzinfo=dt.timezone.utc)
+    status.set_status(temp_db_conn, date, seq=345)
+
+    monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
+                        "get_state_info",
+                        lambda self: OsmosisState(server_sequence, date))
+
+    assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == result

utils/check_server_for_updates.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-from osmium.replication import server
-
-if __name__ == '__main__':
-    if len(sys.argv) != 3:
-        print("Usage: python check_server_for_updates.py <server url> <sequence id>")
-        sys.exit(254)
-
-    seqid = int(sys.argv[2])
-
-    state = server.ReplicationServer(sys.argv[1]).get_state_info()
-
-    if state is None:
-        print("ERROR: Cannot get state from URL %s." % (sys.argv[1], ))
-        sys.exit(253)
-
-    if state.sequence <= seqid:
-        print("Database up to date.")
-        sys.exit(1)
-
-    print("New data available (%i => %i)." % (seqid, state.sequence))
-    sys.exit(0)