Merge pull request #2710 from lonvia/offline-import-mode

Assorted performance improvements for BDD tests

nominatim/clicmd/setup.py

@@ -53,6 +53,8 @@ class SetupAll:
         group.add_argument('--no-updates', action='store_true',
                            help="Do not keep tables that are only needed for "
                                 "updating the database later")
+        group.add_argument('--offline', action='store_true',
+                           help="Do not attempt to load any additional data from the internet")
         group = parser.add_argument_group('Expert options')
         group.add_argument('--ignore-errors', action='store_true',
                            help='Continue import even when errors in SQL are present')
@@ -139,7 +141,7 @@ class SetupAll:
         with connect(args.config.get_libpq_dsn()) as conn:
             refresh.setup_website(webdir, args.config, conn)
 
-        SetupAll._set_database_date(args.config.get_libpq_dsn())
+        SetupAll._finalize_database(args.config.get_libpq_dsn(), args.offline)
 
         return 0
 
@@ -202,15 +204,16 @@ class SetupAll:
 
 
     @staticmethod
-    def _set_database_date(dsn):
+    def _finalize_database(dsn, offline):
         """ Determine the database date and set the status accordingly.
         """
         with connect(dsn) as conn:
-            try:
-                dbdate = status.compute_database_date(conn)
-                status.set_status(conn, dbdate)
-                LOG.info('Database is at %s.', dbdate)
-            except Exception as exc: # pylint: disable=broad-except
-                LOG.error('Cannot determine date of database: %s', exc)
+            if not offline:
+                try:
+                    dbdate = status.compute_database_date(conn)
+                    status.set_status(conn, dbdate)
+                    LOG.info('Database is at %s.', dbdate)
+                except Exception as exc: # pylint: disable=broad-except
+                    LOG.error('Cannot determine date of database: %s', exc)
 
             properties.set_property(conn, 'database_version', version_str())

test/bdd/steps/nominatim_environment.py

@@ -95,6 +95,7 @@ class NominatimEnvironment:
 
         self.test_env = dict(self.default_config)
         self.test_env['NOMINATIM_DATABASE_DSN'] = dsn
+        self.test_env['NOMINATIM_LANGUAGES'] = 'en,de,fr,ja'
         self.test_env['NOMINATIM_FLATNODE_FILE'] = ''
         self.test_env['NOMINATIM_IMPORT_STYLE'] = 'full'
         self.test_env['NOMINATIM_USE_US_TIGER_DATA'] = 'yes'
@@ -172,22 +173,23 @@ class NominatimEnvironment:
 
         self.template_db_done = True
 
-        if self._reuse_or_drop_db(self.template_db):
-            return
-
         self.write_nominatim_config(self.template_db)
 
-        try:
-            # execute nominatim import on an empty file to get the right tables
-            with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
-                fd.write(b'<osm version="0.6"></osm>')
-                fd.flush()
-                self.run_nominatim('import', '--osm-file', fd.name,
-                                             '--osm2pgsql-cache', '1',
-                                             '--ignore-errors')
-        except:
-            self.db_drop_database(self.template_db)
-            raise
+        if not self._reuse_or_drop_db(self.template_db):
+            try:
+                # execute nominatim import on an empty file to get the right tables
+                with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd:
+                    fd.write(b'<osm version="0.6"></osm>')
+                    fd.flush()
+                    self.run_nominatim('import', '--osm-file', fd.name,
+                                                 '--osm2pgsql-cache', '1',
+                                                 '--ignore-errors',
+                                                 '--offline', '--index-noanalyse')
+            except:
+                self.db_drop_database(self.template_db)
+                raise
+
+        self.run_nominatim('refresh', '--functions')
 
 
     def setup_api_db(self):

test/bdd/steps/steps_db_ops.py

@@ -93,9 +93,9 @@ def add_data_to_planet_ways(context):
 def import_and_index_data_from_place_table(context):
     """ Import data previously set up in the place table.
     """
-    context.nominatim.run_nominatim('refresh', '--functions')
     context.nominatim.run_nominatim('import', '--continue', 'load-data',
-                                              '--index-noanalyse', '-q')
+                                              '--index-noanalyse', '-q',
+                                              '--offline')
 
     check_database_integrity(context)