From dc6c4bf22e117aa7f32cd8df280549782be07a7a Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 11 May 2022 15:03:02 +0200 Subject: [PATCH 1/4] add offline import mode In offline mode no attempts are made to download data from the internet. At the moment that only concerns the computation of the database date. It contacts the main API to get the date. --- nominatim/clicmd/setup.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index b643c5ba..7b5f3797 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -53,6 +53,8 @@ class SetupAll: group.add_argument('--no-updates', action='store_true', help="Do not keep tables that are only needed for " "updating the database later") + group.add_argument('--offline', action='store_true', + help="Do not attempt to load any additional data from the internet") group = parser.add_argument_group('Expert options') group.add_argument('--ignore-errors', action='store_true', help='Continue import even when errors in SQL are present') @@ -139,7 +141,7 @@ class SetupAll: with connect(args.config.get_libpq_dsn()) as conn: refresh.setup_website(webdir, args.config, conn) - SetupAll._set_database_date(args.config.get_libpq_dsn()) + SetupAll._finalize_database(args.config.get_libpq_dsn(), args.offline) return 0 @@ -202,15 +204,16 @@ class SetupAll: @staticmethod - def _set_database_date(dsn): + def _finalize_database(dsn, offline): """ Determine the database date and set the status accordingly. """ with connect(dsn) as conn: - try: - dbdate = status.compute_database_date(conn) - status.set_status(conn, dbdate) - LOG.info('Database is at %s.', dbdate) - except Exception as exc: # pylint: disable=broad-except - LOG.error('Cannot determine date of database: %s', exc) + if not offline: + try: + dbdate = status.compute_database_date(conn) + status.set_status(conn, dbdate) + LOG.info('Database is at %s.', dbdate) + except Exception as exc: # pylint: disable=broad-except + LOG.error('Cannot determine date of database: %s', exc) properties.set_property(conn, 'database_version', version_str()) From aa0ae610c6075fdaf9f1790a4ad8ec090a74ebd9 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 11 May 2022 15:33:01 +0200 Subject: [PATCH 2/4] avoid calling OSM servers during bdd tests --- test/bdd/steps/nominatim_environment.py | 3 ++- test/bdd/steps/steps_db_ops.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 70a03e6e..9bf182bc 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -184,7 +184,8 @@ class NominatimEnvironment: fd.flush() self.run_nominatim('import', '--osm-file', fd.name, '--osm2pgsql-cache', '1', - '--ignore-errors') + '--ignore-errors', + '--offline', '--index-noanalyse') except: self.db_drop_database(self.template_db) raise diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py index 4c711b72..5ce416b0 100644 --- a/test/bdd/steps/steps_db_ops.py +++ b/test/bdd/steps/steps_db_ops.py @@ -95,7 +95,8 @@ def import_and_index_data_from_place_table(context): """ context.nominatim.run_nominatim('refresh', '--functions') context.nominatim.run_nominatim('import', '--continue', 'load-data', - '--index-noanalyse', '-q') + '--index-noanalyse', '-q', + '--offline') check_database_integrity(context) From e74e5770294692b4a5bd4a730c974744d5ccfd81 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 11 May 2022 15:50:22 +0200 Subject: [PATCH 3/4] bdd: recreate functions on template DB Avoids calling function refresh on every scenario. The content won't change between runs. --- test/bdd/steps/nominatim_environment.py | 30 ++++++++++++------------- test/bdd/steps/steps_db_ops.py | 1 - 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 9bf182bc..c1a7dc26 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -172,23 +172,23 @@ class NominatimEnvironment: self.template_db_done = True - if self._reuse_or_drop_db(self.template_db): - return - self.write_nominatim_config(self.template_db) - try: - # execute nominatim import on an empty file to get the right tables - with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd: - fd.write(b'') - fd.flush() - self.run_nominatim('import', '--osm-file', fd.name, - '--osm2pgsql-cache', '1', - '--ignore-errors', - '--offline', '--index-noanalyse') - except: - self.db_drop_database(self.template_db) - raise + if not self._reuse_or_drop_db(self.template_db): + try: + # execute nominatim import on an empty file to get the right tables + with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd: + fd.write(b'') + fd.flush() + self.run_nominatim('import', '--osm-file', fd.name, + '--osm2pgsql-cache', '1', + '--ignore-errors', + '--offline', '--index-noanalyse') + except: + self.db_drop_database(self.template_db) + raise + + self.run_nominatim('refresh', '--functions') def setup_api_db(self): diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py index 5ce416b0..63c5120e 100644 --- a/test/bdd/steps/steps_db_ops.py +++ b/test/bdd/steps/steps_db_ops.py @@ -93,7 +93,6 @@ def add_data_to_planet_ways(context): def import_and_index_data_from_place_table(context): """ Import data previously set up in the place table. """ - context.nominatim.run_nominatim('refresh', '--functions') context.nominatim.run_nominatim('import', '--continue', 'load-data', '--index-noanalyse', '-q', '--offline') From f314abcfe1203a95073220357f135fa3c69210fc Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 11 May 2022 16:40:53 +0200 Subject: [PATCH 4/4] bdd: restrict imports to four languages This mainly restricts the number of country names that are loaded. --- test/bdd/steps/nominatim_environment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index c1a7dc26..6b83c2e4 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -95,6 +95,7 @@ class NominatimEnvironment: self.test_env = dict(self.default_config) self.test_env['NOMINATIM_DATABASE_DSN'] = dsn + self.test_env['NOMINATIM_LANGUAGES'] = 'en,de,fr,ja' self.test_env['NOMINATIM_FLATNODE_FILE'] = '' self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full' self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'