From ebae3553e05586d1ddebacf485c92990c36525d0 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 16 Mar 2021 22:13:33 +0100 Subject: [PATCH] bdd: run all setup via nominatim Python library Drops all calls to PHP utility functions. nominatim cli functions are used where possible, to stay as close to the final code as possible with the tests. By removing the PHP calls, the test code now only uses osm2pgsql and the database module from the build directory. --- test/bdd/steps/nominatim_environment.py | 77 +++++++++---------------- test/bdd/steps/steps_db_ops.py | 18 +++++- test/bdd/steps/steps_osm_data.py | 30 ++++++++-- 3 files changed, 67 insertions(+), 58 deletions(-) diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 168334b1..170dd639 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -109,6 +109,22 @@ class NominatimEnvironment: cfg = Configuration(None, self.src_dir / 'settings', environ=self.test_env) refresh.setup_website(Path(self.website_dir.name) / 'website', self.src_dir / 'lib-php', cfg) + def get_libpq_dsn(self): + dsn = self.test_env['NOMINATIM_DATABASE_DSN'] + + def quote_param(param): + key, val = param.split('=') + val = val.replace('\\', '\\\\').replace("'", "\\'") + if ' ' in val: + val = "'" + val + "'" + return key + '=' + val + + if dsn.startswith('pgsql:'): + # Old PHP DSN format. Convert before returning. + return ' '.join([quote_param(p) for p in dsn[6:].split(';')]) + + return dsn + def db_drop_database(self, name): """ Drop the database with the given name. @@ -132,34 +148,16 @@ class NominatimEnvironment: if self._reuse_or_drop_db(self.template_db): return - try: - # call the first part of database setup - self.write_nominatim_config(self.template_db) - self.run_setup_script('create-db', 'setup-db') - # remove external data to speed up indexing for tests - conn = self.connect_database(self.template_db) - cur = conn.cursor() - cur.execute("""select tablename from pg_tables - where tablename in ('gb_postcode', 'us_postcode')""") - for t in cur: - conn.cursor().execute('TRUNCATE TABLE {}'.format(t[0])) - conn.commit() - conn.close() + self.write_nominatim_config(self.template_db) - # execute osm2pgsql import on an empty file to get the right tables + try: + # execute nominatim import on an empty file to get the right tables with tempfile.NamedTemporaryFile(dir='/tmp', suffix='.xml') as fd: fd.write(b'') fd.flush() - self.run_setup_script('import-data', - 'ignore-errors', - 'create-functions', - 'create-tables', - 'create-partition-tables', - 'create-partition-functions', - 'load-data', - 'create-search-indices', - osm_file=fd.name, - osm2pgsql_cache='200') + self.run_nominatim('import', '--osm-file', fd.name, + '--osm2pgsql-cache', '1', + '--ignore-errors') except: self.db_drop_database(self.template_db) raise @@ -179,12 +177,11 @@ class NominatimEnvironment: return testdata = Path('__file__') / '..' / '..' / 'testdb' - self.test_env['NOMINATIM_TIGER_DATA_PATH'] = str((testdata / 'tiger').resolve()) self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve()) try: self.run_nominatim('import', '--osm-file', str(self.api_test_file)) - self.run_setup_script('import-tiger-data') + self.run_nominatim('add-data', '--tiger-data', str((testdata / 'tiger').resolve())) self.run_nominatim('freeze') phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve()) @@ -259,6 +256,9 @@ class NominatimEnvironment: def run_nominatim(self, *cmdline): """ Run the nominatim command-line tool via the library. """ + if self.website_dir is not None: + cmdline = list(cmdline) + ['--project-dir', self.website_dir.name] + cli.nominatim(module_dir='', osm2pgsql_path=str(self.build_dir / 'osm2pgsql' / 'osm2pgsql'), phplib_dir=str(self.src_dir / 'lib-php'), @@ -269,31 +269,6 @@ class NominatimEnvironment: phpcgi_path='', environ=self.test_env) - def run_setup_script(self, *args, **kwargs): - """ Run the Nominatim setup script with the given arguments. - """ - self.run_nominatim_script('setup', *args, **kwargs) - - def run_update_script(self, *args, **kwargs): - """ Run the Nominatim update script with the given arguments. - """ - self.run_nominatim_script('update', *args, **kwargs) - - def run_nominatim_script(self, script, *args, **kwargs): - """ Run one of the Nominatim utility scripts with the given arguments. - """ - cmd = ['/usr/bin/env', 'php', '-Cq'] - cmd.append((Path(self.src_dir) / 'lib-php' / 'admin' / '{}.php'.format(script)).resolve()) - cmd.extend(['--' + x for x in args]) - for k, v in kwargs.items(): - cmd.extend(('--' + k.replace('_', '-'), str(v))) - - if self.website_dir is not None: - cwd = self.website_dir.name - else: - cwd = None - - run_script(cmd, cwd=cwd, env=self.test_env) def copy_from_place(self, db): """ Copy data from place to the placex and location_property_osmline diff --git a/test/bdd/steps/steps_db_ops.py b/test/bdd/steps/steps_db_ops.py index 9d443b43..465eed65 100644 --- a/test/bdd/steps/steps_db_ops.py +++ b/test/bdd/steps/steps_db_ops.py @@ -86,10 +86,24 @@ def import_and_index_data_from_place_table(context): """ Import data previously set up in the place table. """ context.nominatim.copy_from_place(context.db) - context.nominatim.run_setup_script('calculate-postcodes') + + # XXX use tool function as soon as it is ported + with context.db.cursor() as cur: + with (context.nominatim.src_dir / 'lib-sql' / 'postcode_tables.sql').open('r') as fd: + cur.execute(fd.read()) + cur.execute(""" + INSERT INTO location_postcode + (place_id, indexed_status, country_code, postcode, geometry) + SELECT nextval('seq_place'), 1, country_code, + upper(trim (both ' ' from address->'postcode')) as pc, + ST_Centroid(ST_Collect(ST_Centroid(geometry))) + FROM placex + WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%' + AND geometry IS NOT null + GROUP BY country_code, pc""") # Call directly as the refresh function does not include postcodes. - indexer = Indexer(context.nominatim.test_env['NOMINATIM_DATABASE_DSN'][6:], 1) + indexer = Indexer(context.nominatim.get_libpq_dsn(), 1) indexer.index_full(analyse=False) check_database_integrity(context) diff --git a/test/bdd/steps/steps_osm_data.py b/test/bdd/steps/steps_osm_data.py index 844fb274..fb4591bf 100644 --- a/test/bdd/steps/steps_osm_data.py +++ b/test/bdd/steps/steps_osm_data.py @@ -1,6 +1,23 @@ import tempfile import random import os +from pathlib import Path + +from nominatim.tools.exec_utils import run_osm2pgsql + +def get_osm2pgsql_options(nominatim_env, fname, append): + return dict(import_file=fname, + osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'), + osm2pgsql_cache=50, + osm2pgsql_style=str(nominatim_env.src_dir / 'settings' / 'import-extratags.style'), + threads=1, + dsn=nominatim_env.get_libpq_dsn(), + flatnode_file='', + tablespaces=dict(slim_data='', slim_index='', + main_data='', main_index=''), + append=append + ) + def write_opl_file(opl, grid): """ Create a temporary OSM file from OPL and return the file name. It is @@ -52,9 +69,10 @@ def load_osm_file(context): """ # create an OSM file and import it fname = write_opl_file(context.text, context.osm) - context.nominatim.run_setup_script('import-data', osm_file=fname, - osm2pgsql_cache=300) - os.remove(fname) + try: + run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=False)) + finally: + os.remove(fname) ### reintroduce the triggers/indexes we've lost by having osm2pgsql set up place again cur = context.db.cursor() @@ -80,5 +98,7 @@ def update_from_osm_file(context): # create an OSM file and import it fname = write_opl_file(context.text, context.osm) - context.nominatim.run_update_script(import_diff=fname) - os.remove(fname) + try: + run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=True)) + finally: + os.remove(fname)