From 7a874d5b97f65dc1f5a267fd53cc73c2cd5246b6 Mon Sep 17 00:00:00 2001 From: Darkshredder Date: Fri, 12 Mar 2021 10:28:41 +0530 Subject: [PATCH 1/4] Ported createCountryNames() to python and added tests --- lib-php/setup/SetupClass.php | 27 ----------------- nominatim/clicmd/setup.py | 4 +-- nominatim/clicmd/transition.py | 9 +++++- nominatim/tools/database_import.py | 35 +++++++++++++++++++++++ test/python/test_cli.py | 1 + test/python/test_tools_database_import.py | 23 +++++++++++++++ 6 files changed, 69 insertions(+), 30 deletions(-) diff --git a/lib-php/setup/SetupClass.php b/lib-php/setup/SetupClass.php index cf2ac6da..d07adce7 100755 --- a/lib-php/setup/SetupClass.php +++ b/lib-php/setup/SetupClass.php @@ -130,33 +130,6 @@ class SetupFunctions $this->db()->exec($sSQL); } - public function createCountryNames() - { - info('Create search index for default country names'); - - $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); - $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); - $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); - $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); - $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' - .'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; - $sLanguages = getSetting('LANGUAGES'); - if ($sLanguages) { - $sSQL .= 'in '; - $sDelim = '('; - foreach (explode(',', $sLanguages) as $sLang) { - $sSQL .= $sDelim."'name:$sLang'"; - $sDelim = ','; - } - $sSQL .= ')'; - } else { - // all include all simple name tags - $sSQL .= "like 'name:%'"; - } - $sSQL .= ') v'; - $this->pgsqlRunScript($sSQL); - } - /** * Return the connection to the database. * diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index 71980739..056643aa 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -130,8 +130,8 @@ class SetupAll: database_import.create_search_indices(conn, args.config, args.sqllib_dir, drop=args.no_updates) - run_legacy_script('setup.php', '--create-country-names', - nominatim_env=args, throw_on_fail=not args.ignore_errors) + LOG.warning('Create search index for default country names.') + database_import.create_country_names(conn, args.config) webdir = args.project_dir / 'website' LOG.warning('Setup website at %s', webdir) diff --git a/nominatim/clicmd/transition.py b/nominatim/clicmd/transition.py index 0a89cb03..c9341f49 100644 --- a/nominatim/clicmd/transition.py +++ b/nominatim/clicmd/transition.py @@ -43,6 +43,8 @@ class AdminTransition: help='Index the data') group.add_argument('--create-search-indices', action='store_true', help='Create additional indices required for search and update') + group.add_argument('--create-country-names', action='store_true', + help='Create search index for default country names.') group = parser.add_argument_group('Options') group.add_argument('--no-partitions', action='store_true', help='Do not partition search indices') @@ -62,7 +64,7 @@ class AdminTransition: help='File to import') @staticmethod - def run(args): + def run(args): # pylint: disable=too-many-statements from ..tools import database_import, tiger_data from ..tools import refresh @@ -137,3 +139,8 @@ class AdminTransition: args.threads or 1, args.config, args.sqllib_dir) + + if args.create_country_names: + LOG.warning('Create search index for default country names.') + with connect(args.config.get_libpq_dsn()) as conn: + database_import.create_country_names(conn, args.config) diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py index 017c74b6..9d4cad16 100644 --- a/nominatim/tools/database_import.py +++ b/nominatim/tools/database_import.py @@ -306,3 +306,38 @@ def create_search_indices(conn, config, sqllib_dir, drop=False): sql = SQLPreprocessor(conn, config, sqllib_dir) sql.run_sql_file(conn, 'indices.sql', drop=drop) + +def create_country_names(conn, config): + """ Create search index for default country names. + """ + + with conn.cursor() as cur: + cur.execute("""select + getorcreate_country(make_standard_name('uk') + , 'gb')""") + cur.execute("""select getorcreate_country(make_standard_name('united states'), 'us')""") + cur.execute("""select count(*) from + (select getorcreate_country(make_standard_name(country_code), + country_code) from country_name where country_code is not null) as x""") + cur.execute("""select count(*) from + (select getorcreate_country(make_standard_name(name->'name'), + country_code) from country_name where name ? 'name') as x""") + sql_statement = """select count(*) from (select getorcreate_country(make_standard_name(v) + , country_code) from (select country_code, skeys(name) + as k, svals(name) as v from country_name) x where k """ + + languages = config.LANGUAGES + + if languages: + sql_statement += 'in ' + delim = '(' + for language in languages.split(','): + sql_statement += delim + "'name:" + language + "'" + delim = ',' + + sql_statement += ')' + else: + sql_statement += "like 'name:%'" + sql_statement += ') v' + cur.execute(sql_statement) + conn.commit() diff --git a/test/python/test_cli.py b/test/python/test_cli.py index 1f9cd06d..918d8499 100644 --- a/test/python/test_cli.py +++ b/test/python/test_cli.py @@ -95,6 +95,7 @@ def test_import_full(temp_db, mock_func_factory): mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'), mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'), mock_func_factory(nominatim.tools.database_import, 'create_search_indices'), + mock_func_factory(nominatim.tools.database_import, 'create_country_names'), mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_file'), mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim.tools.refresh, 'setup_website'), diff --git a/test/python/test_tools_database_import.py b/test/python/test_tools_database_import.py index 45324834..2e76f5c5 100644 --- a/test/python/test_tools_database_import.py +++ b/test/python/test_tools_database_import.py @@ -200,3 +200,26 @@ def test_load_data(dsn, src_dir, place_row, placex_table, osmline_table, word_ta assert temp_db_cursor.table_rows('placex') == 30 assert temp_db_cursor.table_rows('location_property_osmline') == 1 + +@pytest.mark.parametrize("languages", (False, True)) +def test_create_country_names(temp_db_conn, temp_db_cursor, def_config, + temp_db_with_extensions, monkeypatch, languages): + if languages: + monkeypatch.setenv('NOMINATIM_LANGUAGES', 'fr,en') + temp_db_cursor.execute("""CREATE FUNCTION make_standard_name (name TEXT) + RETURNS TEXT AS $$ SELECT 'a'::TEXT $$ LANGUAGE SQL + """) + temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_country(lookup_word TEXT, + lookup_country_code varchar(2)) + RETURNS INTEGER + AS $$ + BEGIN + INSERT INTO country_name VALUES (5, lookup_word); + RETURN 5; + END; + $$ + LANGUAGE plpgsql; + """) + temp_db_cursor.execute('CREATE TABLE country_name (id int, country_code varchar(2), name hstore)') + database_import.create_country_names(temp_db_conn, def_config) + assert temp_db_cursor.table_rows('country_name') == 4 From 077a8c1f95d2c0a18722f705df4b98d83d6f9385 Mon Sep 17 00:00:00 2001 From: Darkshredder Date: Fri, 12 Mar 2021 18:23:20 +0530 Subject: [PATCH 2/4] refactored tests and made changes to code for easy readibility --- nominatim/tools/database_import.py | 39 +++++++++++------------ test/python/test_tools_database_import.py | 14 +++++--- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py index 9d4cad16..4ba4493c 100644 --- a/nominatim/tools/database_import.py +++ b/nominatim/tools/database_import.py @@ -312,32 +312,29 @@ def create_country_names(conn, config): """ with conn.cursor() as cur: - cur.execute("""select - getorcreate_country(make_standard_name('uk') - , 'gb')""") - cur.execute("""select getorcreate_country(make_standard_name('united states'), 'us')""") - cur.execute("""select count(*) from - (select getorcreate_country(make_standard_name(country_code), - country_code) from country_name where country_code is not null) as x""") - cur.execute("""select count(*) from - (select getorcreate_country(make_standard_name(name->'name'), - country_code) from country_name where name ? 'name') as x""") - sql_statement = """select count(*) from (select getorcreate_country(make_standard_name(v) - , country_code) from (select country_code, skeys(name) - as k, svals(name) as v from country_name) x where k """ + cur.execute("""SELECT getorcreate_country(make_standard_name('uk'), 'gb')""") + cur.execute("""SELECT getorcreate_country(make_standard_name('united states'), 'us')""") + cur.execute("""SELECT COUNT(*) FROM + (SELECT getorcreate_country(make_standard_name(country_code), + country_code) FROM country_name WHERE country_code is not null) AS x""") + cur.execute("""SELECT COUNT(*) FROM + (SELECT getorcreate_country(make_standard_name(name->'name'), country_code) + FROM country_name WHERE name ? 'name') AS x""") + sql_statement = """SELECT COUNT(*) FROM (SELECT getorcreate_country(make_standard_name(v), + country_code) FROM (SELECT country_code, skeys(name) + AS k, svals(name) AS v FROM country_name) x WHERE k""" languages = config.LANGUAGES if languages: - sql_statement += 'in ' - delim = '(' + sql_statement = "{} IN (".format(sql_statement) + delim = '' for language in languages.split(','): - sql_statement += delim + "'name:" + language + "'" - delim = ',' - - sql_statement += ')' + sql_statement = "{}{}'name:{}'".format(sql_statement,delim,language) + delim = ', ' + sql_statement = '{})'.format(sql_statement) else: - sql_statement += "like 'name:%'" - sql_statement += ') v' + sql_statement = "{} LIKE 'name:%'".format(sql_statement) + sql_statement = "{}) v".format(sql_statement) cur.execute(sql_statement) conn.commit() diff --git a/test/python/test_tools_database_import.py b/test/python/test_tools_database_import.py index 2e76f5c5..e2852acb 100644 --- a/test/python/test_tools_database_import.py +++ b/test/python/test_tools_database_import.py @@ -209,17 +209,23 @@ def test_create_country_names(temp_db_conn, temp_db_cursor, def_config, temp_db_cursor.execute("""CREATE FUNCTION make_standard_name (name TEXT) RETURNS TEXT AS $$ SELECT 'a'::TEXT $$ LANGUAGE SQL """) + temp_db_cursor.execute('CREATE TABLE country_name (country_code varchar(2), name hstore)') + temp_db_cursor.execute('CREATE TABLE word (code varchar(2))') + temp_db_cursor.execute("""INSERT INTO country_name VALUES ('us', + '"name"=>"us","name:af"=>"us"')""") temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_country(lookup_word TEXT, - lookup_country_code varchar(2)) + lookup_country_code varchar(2)) RETURNS INTEGER AS $$ BEGIN - INSERT INTO country_name VALUES (5, lookup_word); + INSERT INTO word VALUES (lookup_country_code); RETURN 5; END; $$ LANGUAGE plpgsql; """) - temp_db_cursor.execute('CREATE TABLE country_name (id int, country_code varchar(2), name hstore)') database_import.create_country_names(temp_db_conn, def_config) - assert temp_db_cursor.table_rows('country_name') == 4 + if languages: + assert temp_db_cursor.table_rows('word') == 4 + else: + assert temp_db_cursor.table_rows('word') == 5 From b108bd1c1ef0b8166d9642db77ccbc9f3dca1401 Mon Sep 17 00:00:00 2001 From: Darkshredder Date: Fri, 12 Mar 2021 18:28:47 +0530 Subject: [PATCH 3/4] Linting fix --- nominatim/tools/database_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py index 4ba4493c..433cd8af 100644 --- a/nominatim/tools/database_import.py +++ b/nominatim/tools/database_import.py @@ -330,7 +330,7 @@ def create_country_names(conn, config): sql_statement = "{} IN (".format(sql_statement) delim = '' for language in languages.split(','): - sql_statement = "{}{}'name:{}'".format(sql_statement,delim,language) + sql_statement = "{}{}'name:{}'".format(sql_statement, delim, language) delim = ', ' sql_statement = '{})'.format(sql_statement) else: From f356a75a2477d8638204b864df7403d2488c0e9a Mon Sep 17 00:00:00 2001 From: Darkshredder Date: Sun, 14 Mar 2021 15:02:30 +0530 Subject: [PATCH 4/4] Add setup.php --- lib-php/admin/setup.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-php/admin/setup.php b/lib-php/admin/setup.php index 0b008495..7523527a 100644 --- a/lib-php/admin/setup.php +++ b/lib-php/admin/setup.php @@ -198,7 +198,7 @@ if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { $bDidSomething = true; - $oSetup->createCountryNames($aCMDResult); + run(clone($oNominatimCmd))->addParams('transition', '--create-country-names'); } if ($aCMDResult['setup-website'] || $aCMDResult['all']) {