Merge pull request #2710 from lonvia/offline-import-mode

Assorted performance improvements for BDD tests

nominatim/clicmd/setup.py

@@ -53,6 +53,8 @@ class SetupAll:
         group.add_argument('--no-updates', action='store_true',
                            help="Do not keep tables that are only needed for "
                                 "updating the database later")
+        group.add_argument('--offline', action='store_true',
+                           help="Do not attempt to load any additional data from the internet")
         group = parser.add_argument_group('Expert options')
         group.add_argument('--ignore-errors', action='store_true',
                            help='Continue import even when errors in SQL are present')
@@ -139,7 +141,7 @@ class SetupAll:
         with connect(args.config.get_libpq_dsn()) as conn:
             refresh.setup_website(webdir, args.config, conn)
 
-        SetupAll._set_database_date(args.config.get_libpq_dsn())
+        SetupAll._finalize_database(args.config.get_libpq_dsn(), args.offline)
 
         return 0
 
@@ -202,10 +204,11 @@ class SetupAll:
 
 
     @staticmethod
-    def _set_database_date(dsn):
+    def _finalize_database(dsn, offline):
         """ Determine the database date and set the status accordingly.
         """
         with connect(dsn) as conn:
+            if not offline:
                 try:
                     dbdate = status.compute_database_date(conn)
                     status.set_status(conn, dbdate)

test/bdd/steps/nominatim_environment.py

@@ -95,6 +95,7 @@ class NominatimEnvironment:
 
         self.test_env = dict(self.default_config)
         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
+        self.test_env['NOMINATIM_LANGUAGES'] = 'en,de,fr,ja'
         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
@@ -172,11 +173,9 @@ class NominatimEnvironment:
 
         self.template_db_done = True
 
-        if self._reuse_or_drop_db(self.template_db):
-            return
-
         self.write_nominatim_config(self.template_db)
 
+        if not self._reuse_or_drop_db(self.template_db):
             try:
                 # execute nominatim import on an empty file to get the right tables
                 with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
@@ -184,11 +183,14 @@ class NominatimEnvironment:
                     fd.flush()
                     self.run_nominatim('import', '--osm-file', fd.name,
                                                  '--osm2pgsql-cache', '1',
-                                             '--ignore-errors')
+                                                 '--ignore-errors',
+                                                 '--offline', '--index-noanalyse')
             except:
                 self.db_drop_database(self.template_db)
                 raise
 
+        self.run_nominatim('refresh', '--functions')
+
 
     def setup_api_db(self):
         """ Setup a test against the API test database.

test/bdd/steps/steps_db_ops.py

@@ -93,9 +93,9 @@ def add_data_to_planet_ways(context):
 def import_and_index_data_from_place_table(context):
     """ Import data previously set up in the place table.
     """
-    context.nominatim.run_nominatim('refresh', '--functions')
     context.nominatim.run_nominatim('import', '--continue', 'load-data',
-                                              '--index-noanalyse', '-q')
+                                              '--index-noanalyse', '-q',
+                                              '--offline')
 
     check_database_integrity(context)