osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-11-27 00:49:55 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2212 from darkshredder/country-name

Browse Source 
Ported createCountryNames() to python and Added tests
This commit is contained in:
Sarah Hoffmann 2021-03-15 09:36:06 +01:00 committed by GitHub
commit 81a6b746b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 73 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib-php/admin/setup.php
View File

@ -198,7 +198,7 @@ if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createCountryNames($aCMDResult);
run(clone($oNominatimCmd))->addParams('transition', '--create-country-names');
}
if ($aCMDResult['setup-website'] || $aCMDResult['all']) {

27
lib-php/setup/SetupClass.php
View File

@ -130,33 +130,6 @@ class SetupFunctions
$this->db()->exec($sSQL);
}
public function createCountryNames()
{
info('Create search index for default country names');
$this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')");
$this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')");
$this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x');
$this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x");
$sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),'
.'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
$sLanguages = getSetting('LANGUAGES');
if ($sLanguages) {
$sSQL .= 'in ';
$sDelim = '(';
foreach (explode(',', $sLanguages) as $sLang) {
$sSQL .= $sDelim."'name:$sLang'";
$sDelim = ',';
}
$sSQL .= ')';
} else {
// all include all simple name tags
$sSQL .= "like 'name:%'";
}
$sSQL .= ') v';
$this->pgsqlRunScript($sSQL);
}
/**
* Return the connection to the database.
*

4
nominatim/clicmd/setup.py
View File

@ -130,8 +130,8 @@ class SetupAll:
database_import.create_search_indices(conn, args.config,
args.sqllib_dir,
drop=args.no_updates)
run_legacy_script('setup.php', '--create-country-names',
nominatim_env=args, throw_on_fail=not args.ignore_errors)
LOG.warning('Create search index for default country names.')
database_import.create_country_names(conn, args.config)
webdir = args.project_dir / 'website'
LOG.warning('Setup website at %s', webdir)

9
nominatim/clicmd/transition.py
View File

@ -43,6 +43,8 @@ class AdminTransition:
help='Index the data')
group.add_argument('--create-search-indices', action='store_true',
help='Create additional indices required for search and update')
group.add_argument('--create-country-names', action='store_true',
help='Create search index for default country names.')
group = parser.add_argument_group('Options')
group.add_argument('--no-partitions', action='store_true',
help='Do not partition search indices')
@ -62,7 +64,7 @@ class AdminTransition:
help='File to import')
@staticmethod
def run(args):
def run(args): # pylint: disable=too-many-statements
from ..tools import database_import, tiger_data
from ..tools import refresh
@ -137,3 +139,8 @@ class AdminTransition:
args.threads or 1,
args.config,
args.sqllib_dir)
if args.create_country_names:
LOG.warning('Create search index for default country names.')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_country_names(conn, args.config)

32
nominatim/tools/database_import.py
View File

@ -306,3 +306,35 @@ def create_search_indices(conn, config, sqllib_dir, drop=False):
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'indices.sql', drop=drop)
def create_country_names(conn, config):
""" Create search index for default country names.
"""
with conn.cursor() as cur:
cur.execute("""SELECT getorcreate_country(make_standard_name('uk'), 'gb')""")
cur.execute("""SELECT getorcreate_country(make_standard_name('united states'), 'us')""")
cur.execute("""SELECT COUNT(*) FROM
(SELECT getorcreate_country(make_standard_name(country_code),
country_code) FROM country_name WHERE country_code is not null) AS x""")
cur.execute("""SELECT COUNT(*) FROM
(SELECT getorcreate_country(make_standard_name(name->'name'), country_code)
FROM country_name WHERE name ? 'name') AS x""")
sql_statement = """SELECT COUNT(*) FROM (SELECT getorcreate_country(make_standard_name(v),
country_code) FROM (SELECT country_code, skeys(name)
AS k, svals(name) AS v FROM country_name) x WHERE k"""
languages = config.LANGUAGES
if languages:
sql_statement = "{} IN (".format(sql_statement)
delim = ''
for language in languages.split(','):
sql_statement = "{}{}'name:{}'".format(sql_statement, delim, language)
delim = ', '
sql_statement = '{})'.format(sql_statement)
else:
sql_statement = "{} LIKE 'name:%'".format(sql_statement)
sql_statement = "{}) v".format(sql_statement)
cur.execute(sql_statement)
conn.commit()

1
test/python/test_cli.py
View File

@ -95,6 +95,7 @@ def test_import_full(temp_db, mock_func_factory):
mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'),
mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'),
mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim.tools.database_import, 'create_country_names'),
mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_file'),
mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim.tools.refresh, 'setup_website'),

29
test/python/test_tools_database_import.py
View File

@ -200,3 +200,32 @@ def test_load_data(dsn, src_dir, place_row, placex_table, osmline_table, word_ta
assert temp_db_cursor.table_rows('placex') == 30
assert temp_db_cursor.table_rows('location_property_osmline') == 1
@pytest.mark.parametrize("languages", (False, True))
def test_create_country_names(temp_db_conn, temp_db_cursor, def_config,
temp_db_with_extensions, monkeypatch, languages):
if languages:
monkeypatch.setenv('NOMINATIM_LANGUAGES', 'fr,en')
temp_db_cursor.execute("""CREATE FUNCTION make_standard_name (name TEXT)
RETURNS TEXT AS $$ SELECT 'a'::TEXT $$ LANGUAGE SQL
""")
temp_db_cursor.execute('CREATE TABLE country_name (country_code varchar(2), name hstore)')
temp_db_cursor.execute('CREATE TABLE word (code varchar(2))')
temp_db_cursor.execute("""INSERT INTO country_name VALUES ('us',
'"name"=>"us","name:af"=>"us"')""")
temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION getorcreate_country(lookup_word TEXT,
lookup_country_code varchar(2))
RETURNS INTEGER
AS $$
BEGIN
INSERT INTO word VALUES (lookup_country_code);
RETURN 5;
END;
$$
LANGUAGE plpgsql;
""")
database_import.create_country_names(temp_db_conn, def_config)
if languages:
assert temp_db_cursor.table_rows('word') == 4
else:
assert temp_db_cursor.table_rows('word') == 5