mirror of
https://github.com/osm-search/Nominatim.git
synced 2024-12-26 14:36:23 +03:00
Merge pull request #2181 from lonvia/port-more-tool-functions-to-python
Port more tool functions to python
This commit is contained in:
Sarah Hoffmann
GitHub
commit
72b01148d2
@ -240,16 +240,6 @@ class DB
|
||||
return ($this->getOne($sSQL, array(':tablename' => $sTableName)) == 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list of table names in the database
|
||||
*
|
||||
* @return array[]
|
||||
*/
|
||||
public function getListOfTables()
|
||||
{
|
||||
return $this->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'");
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a table. Returns true if deleted or didn't exist.
|
||||
*
|
||||
@ -262,76 +252,6 @@ class DB
|
||||
return $this->exec('DROP TABLE IF EXISTS '.$sTableName.' CASCADE') == 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an index exists in the database. Optional filtered by tablename
|
||||
*
|
||||
* @param string $sTableName
|
||||
*
|
||||
* @return boolean
|
||||
*/
|
||||
public function indexExists($sIndexName, $sTableName = null)
|
||||
{
|
||||
return in_array($sIndexName, $this->getListOfIndices($sTableName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a list of index names in the database, optional filtered by tablename
|
||||
*
|
||||
* @param string $sTableName
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getListOfIndices($sTableName = null)
|
||||
{
|
||||
// table_name | index_name | column_name
|
||||
// -----------------------+---------------------------------+--------------
|
||||
// country_name | idx_country_name_country_code | country_code
|
||||
// country_osm_grid | idx_country_osm_grid_geometry | geometry
|
||||
// import_polygon_delete | idx_import_polygon_delete_osmid | osm_id
|
||||
// import_polygon_delete | idx_import_polygon_delete_osmid | osm_type
|
||||
// import_polygon_error | idx_import_polygon_error_osmid | osm_id
|
||||
// import_polygon_error | idx_import_polygon_error_osmid | osm_type
|
||||
$sSql = <<< END
|
||||
SELECT
|
||||
t.relname as table_name,
|
||||
i.relname as index_name,
|
||||
a.attname as column_name
|
||||
FROM
|
||||
pg_class t,
|
||||
pg_class i,
|
||||
pg_index ix,
|
||||
pg_attribute a
|
||||
WHERE
|
||||
t.oid = ix.indrelid
|
||||
and i.oid = ix.indexrelid
|
||||
and a.attrelid = t.oid
|
||||
and a.attnum = ANY(ix.indkey)
|
||||
and t.relkind = 'r'
|
||||
and i.relname NOT LIKE 'pg_%'
|
||||
FILTERS
|
||||
ORDER BY
|
||||
t.relname,
|
||||
i.relname,
|
||||
a.attname
|
||||
END;
|
||||
|
||||
$aRows = null;
|
||||
if ($sTableName) {
|
||||
$sSql = str_replace('FILTERS', 'and t.relname = :tablename', $sSql);
|
||||
$aRows = $this->getAll($sSql, array(':tablename' => $sTableName));
|
||||
} else {
|
||||
$sSql = str_replace('FILTERS', '', $sSql);
|
||||
$aRows = $this->getAll($sSql);
|
||||
}
|
||||
|
||||
$aIndexNames = array_unique(array_map(function ($aRow) {
|
||||
return $aRow['index_name'];
|
||||
}, $aRows));
|
||||
sort($aIndexNames);
|
||||
|
||||
return $aIndexNames;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries to connect to the database but on failure doesn't throw an exception.
|
||||
*
|
||||
|
||||
@ -5,197 +5,6 @@ require_once(CONST_LibDir.'/init-cmd.php');
|
||||
|
||||
loadSettings(getcwd());
|
||||
|
||||
$term_colors = array(
|
||||
'green' => "\033[92m",
|
||||
'red' => "\x1B[31m",
|
||||
'normal' => "\033[0m"
|
||||
);
|
||||
|
||||
$print_success = function ($message = 'OK') use ($term_colors) {
|
||||
echo $term_colors['green'].$message.$term_colors['normal']."\n";
|
||||
};
|
||||
|
||||
$print_fail = function ($message = 'Failed') use ($term_colors) {
|
||||
echo $term_colors['red'].$message.$term_colors['normal']."\n";
|
||||
};
|
||||
|
||||
$oDB = new Nominatim\DB;
|
||||
|
||||
|
||||
function isReverseOnlyInstallation()
|
||||
{
|
||||
global $oDB;
|
||||
return !$oDB->tableExists('search_name');
|
||||
}
|
||||
|
||||
// Check (guess) if the setup.php included --drop
|
||||
function isNoUpdateInstallation()
|
||||
{
|
||||
global $oDB;
|
||||
return $oDB->tableExists('placex') && !$oDB->tableExists('planet_osm_rels') ;
|
||||
}
|
||||
|
||||
|
||||
echo 'Checking database got created ... ';
|
||||
if ($oDB->checkConnection()) {
|
||||
$print_success();
|
||||
} else {
|
||||
$print_fail();
|
||||
echo <<< END
|
||||
Hints:
|
||||
* Is the database server started?
|
||||
* Check the NOMINATIM_DATABASE_DSN variable in your local .env
|
||||
* Try connecting to the database with the same settings
|
||||
|
||||
END;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
||||
echo 'Checking nominatim.so module installed ... ';
|
||||
$sStandardWord = $oDB->getOne("SELECT make_standard_name('a')");
|
||||
if ($sStandardWord === 'a') {
|
||||
$print_success();
|
||||
} else {
|
||||
$print_fail();
|
||||
echo <<< END
|
||||
The Postgresql extension nominatim.so was not found in the database.
|
||||
Hints:
|
||||
* Check the output of the CMmake/make installation step
|
||||
* Does nominatim.so exist?
|
||||
* Does nominatim.so exist on the database server?
|
||||
* Can nominatim.so be accessed by the database user?
|
||||
|
||||
END;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (!isNoUpdateInstallation()) {
|
||||
echo 'Checking place table ... ';
|
||||
if ($oDB->tableExists('place')) {
|
||||
$print_success();
|
||||
} else {
|
||||
$print_fail();
|
||||
echo <<< END
|
||||
* The import didn't finish.
|
||||
Hints:
|
||||
* Check the output of the utils/setup.php you ran.
|
||||
Usually the osm2pgsql step failed. Check for errors related to
|
||||
* the file you imported not containing any places
|
||||
* harddrive full
|
||||
* out of memory (RAM)
|
||||
* osm2pgsql killed by other scripts, for consuming to much memory
|
||||
|
||||
END;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
echo 'Checking indexing status ... ';
|
||||
$iUnindexed = $oDB->getOne('SELECT count(*) FROM placex WHERE indexed_status > 0');
|
||||
if ($iUnindexed == 0) {
|
||||
$print_success();
|
||||
} else {
|
||||
$print_fail();
|
||||
echo <<< END
|
||||
The indexing didn't finish. There is still $iUnindexed places. See the
|
||||
question 'Can a stopped/killed import process be resumed?' in the
|
||||
troubleshooting guide.
|
||||
|
||||
END;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
echo "Search index creation\n";
|
||||
$aExpectedIndices = array(
|
||||
// sql/indices.src.sql
|
||||
'idx_word_word_id',
|
||||
'idx_place_addressline_address_place_id',
|
||||
'idx_placex_rank_search',
|
||||
'idx_placex_rank_address',
|
||||
'idx_placex_parent_place_id',
|
||||
'idx_placex_geometry_reverse_lookuppolygon',
|
||||
'idx_placex_geometry_reverse_placenode',
|
||||
'idx_osmline_parent_place_id',
|
||||
'idx_osmline_parent_osm_id',
|
||||
'idx_postcode_id',
|
||||
'idx_postcode_postcode'
|
||||
);
|
||||
if (!isReverseOnlyInstallation()) {
|
||||
$aExpectedIndices = array_merge($aExpectedIndices, array(
|
||||
// sql/indices_search.src.sql
|
||||
'idx_search_name_nameaddress_vector',
|
||||
'idx_search_name_name_vector',
|
||||
'idx_search_name_centroid'
|
||||
));
|
||||
}
|
||||
if (!isNoUpdateInstallation()) {
|
||||
$aExpectedIndices = array_merge($aExpectedIndices, array(
|
||||
'idx_placex_pendingsector',
|
||||
'idx_location_area_country_place_id',
|
||||
'idx_place_osm_unique',
|
||||
));
|
||||
}
|
||||
|
||||
foreach ($aExpectedIndices as $sExpectedIndex) {
|
||||
echo "Checking index $sExpectedIndex ... ";
|
||||
if ($oDB->indexExists($sExpectedIndex)) {
|
||||
$print_success();
|
||||
} else {
|
||||
$print_fail();
|
||||
echo <<< END
|
||||
Hints:
|
||||
* Run './utils/setup.php --create-search-indices --ignore-errors' to
|
||||
create missing indices.
|
||||
|
||||
END;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
echo 'Checking search indices are valid ... ';
|
||||
$sSQL = <<< END
|
||||
SELECT relname
|
||||
FROM pg_class, pg_index
|
||||
WHERE pg_index.indisvalid = false
|
||||
AND pg_index.indexrelid = pg_class.oid;
|
||||
END;
|
||||
$aInvalid = $oDB->getCol($sSQL);
|
||||
if (empty($aInvalid)) {
|
||||
$print_success();
|
||||
} else {
|
||||
$print_fail();
|
||||
echo <<< END
|
||||
At least one index is invalid. That can happen, e.g. when index creation was
|
||||
disrupted and later restarted. You should delete the affected indices and
|
||||
run the index stage of setup again.
|
||||
See the question 'Can a stopped/killed import process be resumed?' in the
|
||||
troubleshooting guide.
|
||||
Affected indices:
|
||||
END;
|
||||
echo join(', ', $aInvalid) . "\n";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (getSettingBool('USE_US_TIGER_DATA')) {
|
||||
echo 'Checking TIGER table exists ... ';
|
||||
if ($oDB->tableExists('location_property_tiger')) {
|
||||
$print_success();
|
||||
} else {
|
||||
$print_fail();
|
||||
echo <<< END
|
||||
Table 'location_property_tiger' does not exist. Run the TIGER data
|
||||
import again.
|
||||
|
||||
END;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
exit(0);
|
||||
(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
|
||||
->addParams('admin', '--check-database')
|
||||
->run();
|
||||
|
||||
@ -132,24 +132,6 @@ function addQuotes($s)
|
||||
return "'".$s."'";
|
||||
}
|
||||
|
||||
function fwriteConstDef($rFile, $sConstName, $value)
|
||||
{
|
||||
$sEscapedValue;
|
||||
|
||||
if (is_bool($value)) {
|
||||
$sEscapedValue = $value ? 'true' : 'false';
|
||||
} elseif (is_numeric($value)) {
|
||||
$sEscapedValue = strval($value);
|
||||
} elseif (!$value) {
|
||||
$sEscapedValue = 'false';
|
||||
} else {
|
||||
$sEscapedValue = addQuotes(str_replace("'", "\\'", (string)$value));
|
||||
}
|
||||
|
||||
fwrite($rFile, "@define('CONST_$sConstName', $sEscapedValue);\n");
|
||||
}
|
||||
|
||||
|
||||
function parseLatLon($sQuery)
|
||||
{
|
||||
$sFound = null;
|
||||
@ -226,17 +208,6 @@ function parseLatLon($sQuery)
|
||||
return array($sFound, $fQueryLat, $fQueryLon);
|
||||
}
|
||||
|
||||
function createPointsAroundCenter($fLon, $fLat, $fRadius)
|
||||
{
|
||||
$iSteps = max(8, min(100, ($fRadius * 40000)^2));
|
||||
$fStepSize = (2*pi())/$iSteps;
|
||||
$aPolyPoints = array();
|
||||
for ($f = 0; $f < 2*pi(); $f += $fStepSize) {
|
||||
$aPolyPoints[] = array('', $fLon+($fRadius*sin($f)), $fLat+($fRadius*cos($f)) );
|
||||
}
|
||||
return $aPolyPoints;
|
||||
}
|
||||
|
||||
function closestHouseNumber($aRow)
|
||||
{
|
||||
$fHouse = $aRow['startnumber']
|
||||
@ -256,25 +227,3 @@ function closestHouseNumber($aRow)
|
||||
|
||||
return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']);
|
||||
}
|
||||
|
||||
function getSearchRankLabel($iRank)
|
||||
{
|
||||
if (!isset($iRank)) return 'unknown';
|
||||
if ($iRank < 2) return 'continent';
|
||||
if ($iRank < 4) return 'sea';
|
||||
if ($iRank < 8) return 'country';
|
||||
if ($iRank < 12) return 'state';
|
||||
if ($iRank < 16) return 'county';
|
||||
if ($iRank == 16) return 'city';
|
||||
if ($iRank == 17) return 'town / island';
|
||||
if ($iRank == 18) return 'village / hamlet';
|
||||
if ($iRank == 20) return 'suburb';
|
||||
if ($iRank == 21) return 'postcode area';
|
||||
if ($iRank == 22) return 'croft / farm / locality / islet';
|
||||
if ($iRank == 23) return 'postcode area';
|
||||
if ($iRank == 25) return 'postcode point';
|
||||
if ($iRank == 26) return 'street / major landmark';
|
||||
if ($iRank == 27) return 'minory street / path';
|
||||
if ($iRank == 28) return 'house / building';
|
||||
return 'other: ' . $iRank;
|
||||
}
|
||||
|
||||
@ -657,50 +657,7 @@ class SetupFunctions
|
||||
|
||||
public function drop()
|
||||
{
|
||||
info('Drop tables only required for updates');
|
||||
|
||||
// The implementation is potentially a bit dangerous because it uses
|
||||
// a positive selection of tables to keep, and deletes everything else.
|
||||
// Including any tables that the unsuspecting user might have manually
|
||||
// created. USE AT YOUR OWN PERIL.
|
||||
// tables we want to keep. everything else goes.
|
||||
$aKeepTables = array(
|
||||
'*columns',
|
||||
'import_polygon_*',
|
||||
'import_status',
|
||||
'place_addressline',
|
||||
'location_postcode',
|
||||
'location_property*',
|
||||
'placex',
|
||||
'search_name',
|
||||
'seq_*',
|
||||
'word',
|
||||
'query_log',
|
||||
'new_query_log',
|
||||
'spatial_ref_sys',
|
||||
'country_name',
|
||||
'place_classtype_*',
|
||||
'country_osm_grid'
|
||||
);
|
||||
|
||||
$aDropTables = array();
|
||||
$aHaveTables = $this->db()->getListOfTables();
|
||||
|
||||
foreach ($aHaveTables as $sTable) {
|
||||
$bFound = false;
|
||||
foreach ($aKeepTables as $sKeep) {
|
||||
if (fnmatch($sKeep, $sTable)) {
|
||||
$bFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!$bFound) array_push($aDropTables, $sTable);
|
||||
}
|
||||
foreach ($aDropTables as $sDrop) {
|
||||
$this->dropTable($sDrop);
|
||||
}
|
||||
|
||||
$this->removeFlatnodeFile();
|
||||
(clone($this->oNominatimCmd))->addParams('freeze')->run();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -710,48 +667,7 @@ class SetupFunctions
|
||||
*/
|
||||
public function setupWebsite()
|
||||
{
|
||||
if (!is_dir(CONST_InstallDir.'/website')) {
|
||||
info('Creating directory for website scripts at: '.CONST_InstallDir.'/website');
|
||||
mkdir(CONST_InstallDir.'/website');
|
||||
}
|
||||
|
||||
$aScripts = array(
|
||||
'deletable.php',
|
||||
'details.php',
|
||||
'lookup.php',
|
||||
'polygons.php',
|
||||
'reverse.php',
|
||||
'search.php',
|
||||
'status.php'
|
||||
);
|
||||
|
||||
foreach ($aScripts as $sScript) {
|
||||
$rFile = fopen(CONST_InstallDir.'/website/'.$sScript, 'w');
|
||||
|
||||
fwrite($rFile, "<?php\n\n");
|
||||
fwrite($rFile, '@define(\'CONST_Debug\', $_GET[\'debug\'] ?? false);'."\n\n");
|
||||
|
||||
fwriteConstDef($rFile, 'LibDir', CONST_LibDir);
|
||||
fwriteConstDef($rFile, 'Database_DSN', getSetting('DATABASE_DSN'));
|
||||
fwriteConstDef($rFile, 'Default_Language', getSetting('DEFAULT_LANGUAGE'));
|
||||
fwriteConstDef($rFile, 'Log_DB', getSettingBool('LOG_DB'));
|
||||
fwriteConstDef($rFile, 'Log_File', getSetting('LOG_FILE'));
|
||||
fwriteConstDef($rFile, 'Max_Word_Frequency', (int)getSetting('MAX_WORD_FREQUENCY'));
|
||||
fwriteConstDef($rFile, 'NoAccessControl', getSettingBool('CORS_NOACCESSCONTROL'));
|
||||
fwriteConstDef($rFile, 'Places_Max_ID_count', (int)getSetting('LOOKUP_MAX_COUNT'));
|
||||
fwriteConstDef($rFile, 'PolygonOutput_MaximumTypes', getSetting('POLYGON_OUTPUT_MAX_TYPES'));
|
||||
fwriteConstDef($rFile, 'Search_BatchMode', getSettingBool('SEARCH_BATCH_MODE'));
|
||||
fwriteConstDef($rFile, 'Search_NameOnlySearchFrequencyThreshold', getSetting('SEARCH_NAME_ONLY_THRESHOLD'));
|
||||
fwriteConstDef($rFile, 'Term_Normalization_Rules', getSetting('TERM_NORMALIZATION'));
|
||||
fwriteConstDef($rFile, 'Use_Aux_Location_data', getSettingBool('USE_AUX_LOCATION_DATA'));
|
||||
fwriteConstDef($rFile, 'Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA'));
|
||||
fwriteConstDef($rFile, 'MapIcon_URL', getSetting('MAPICON_URL'));
|
||||
|
||||
fwrite($rFile, 'require_once(\''.CONST_LibDir.'/website/'.$sScript."');\n");
|
||||
fclose($rFile);
|
||||
|
||||
chmod(CONST_InstallDir.'/website/'.$sScript, 0755);
|
||||
}
|
||||
(clone($this->oNominatimCmd))->addParams('refresh', '--website')->run();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -53,7 +53,7 @@ if ($sOsmType && $iOsmId > 0) {
|
||||
|
||||
// Be nice about our error messages for broken geometry
|
||||
|
||||
if (!$sPlaceId) {
|
||||
if (!$sPlaceId && $oDB->tableExists('import_polygon_error')) {
|
||||
$sSQL = 'SELECT ';
|
||||
$sSQL .= ' osm_type, ';
|
||||
$sSQL .= ' osm_id, ';
|
||||
@ -144,7 +144,6 @@ if (!$aPointDetails) {
|
||||
}
|
||||
|
||||
$aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber'];
|
||||
$aPointDetails['rank_search_label'] = getSearchRankLabel($aPointDetails['rank_search']); // only used in HTML format
|
||||
|
||||
// Get all alternative names (languages, etc)
|
||||
$sSQL = 'SELECT (each(name)).key,(each(name)).value FROM placex ';
|
||||
|
||||
@ -173,27 +173,6 @@ class SetupAll:
|
||||
return run_legacy_script(*params, nominatim_env=args)
|
||||
|
||||
|
||||
class SetupFreeze:
|
||||
"""\
|
||||
Make database read-only.
|
||||
|
||||
About half of data in the Nominatim database is kept only to be able to
|
||||
keep the data up-to-date with new changes made in OpenStreetMap. This
|
||||
command drops all this data and only keeps the part needed for geocoding
|
||||
itself.
|
||||
|
||||
This command has the same effect as the `--no-updates` option for imports.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
pass # No options
|
||||
|
||||
@staticmethod
|
||||
def run(args):
|
||||
return run_legacy_script('setup.php', '--drop', nominatim_env=args)
|
||||
|
||||
|
||||
class SetupSpecialPhrases:
|
||||
"""\
|
||||
Maintain special phrases.
|
||||
@ -352,7 +331,7 @@ def nominatim(**kwargs):
|
||||
parser = CommandlineParser('nominatim', nominatim.__doc__)
|
||||
|
||||
parser.add_subcommand('import', SetupAll)
|
||||
parser.add_subcommand('freeze', SetupFreeze)
|
||||
parser.add_subcommand('freeze', clicmd.SetupFreeze)
|
||||
parser.add_subcommand('replication', clicmd.UpdateReplication)
|
||||
|
||||
parser.add_subcommand('special-phrases', SetupSpecialPhrases)
|
||||
|
||||
@ -7,3 +7,4 @@ from .api import APISearch, APIReverse, APILookup, APIDetails, APIStatus
|
||||
from .index import UpdateIndex
|
||||
from .refresh import UpdateRefresh
|
||||
from .admin import AdminFuncs
|
||||
from .freeze import SetupFreeze
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
"""
|
||||
Implementation of the 'admin' subcommand.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from ..tools.exec_utils import run_legacy_script
|
||||
from ..db.connection import connect
|
||||
|
||||
@ -9,6 +11,8 @@ from ..db.connection import connect
|
||||
# Using non-top-level imports to avoid eventually unused imports.
|
||||
# pylint: disable=E0012,C0415
|
||||
|
||||
LOG = logging.getLogger()
|
||||
|
||||
class AdminFuncs:
|
||||
"""\
|
||||
Analyse and maintain the database.
|
||||
@ -39,14 +43,17 @@ class AdminFuncs:
|
||||
|
||||
@staticmethod
|
||||
def run(args):
|
||||
from ..tools import admin
|
||||
if args.warm:
|
||||
AdminFuncs._warm(args)
|
||||
|
||||
if args.check_database:
|
||||
run_legacy_script('check_import_finished.php', nominatim_env=args)
|
||||
LOG.warning('Checking database')
|
||||
from ..tools import check_database
|
||||
return check_database.check_database(args.config)
|
||||
|
||||
if args.analyse_indexing:
|
||||
LOG.warning('Analysing performance of indexing function')
|
||||
from ..tools import admin
|
||||
conn = connect(args.config.get_libpq_dsn())
|
||||
admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id)
|
||||
conn.close()
|
||||
@ -56,6 +63,7 @@ class AdminFuncs:
|
||||
|
||||
@staticmethod
|
||||
def _warm(args):
|
||||
LOG.warning('Warming database caches')
|
||||
params = ['warm.php']
|
||||
if args.target == 'reverse':
|
||||
params.append('--reverse-only')
|
||||
|
||||
37
nominatim/clicmd/freeze.py
Normal file
37
nominatim/clicmd/freeze.py
Normal file
@ -0,0 +1,37 @@
|
||||
"""
|
||||
Implementation of the 'freeze' subcommand.
|
||||
"""
|
||||
|
||||
from ..db.connection import connect
|
||||
|
||||
# Do not repeat documentation of subcommand classes.
|
||||
# pylint: disable=C0111
|
||||
# Using non-top-level imports to avoid eventually unused imports.
|
||||
# pylint: disable=E0012,C0415
|
||||
|
||||
class SetupFreeze:
|
||||
"""\
|
||||
Make database read-only.
|
||||
|
||||
About half of data in the Nominatim database is kept only to be able to
|
||||
keep the data up-to-date with new changes made in OpenStreetMap. This
|
||||
command drops all this data and only keeps the part needed for geocoding
|
||||
itself.
|
||||
|
||||
This command has the same effect as the `--no-updates` option for imports.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
pass # No options
|
||||
|
||||
@staticmethod
|
||||
def run(args):
|
||||
from ..tools import freeze
|
||||
|
||||
conn = connect(args.config.get_libpq_dsn())
|
||||
freeze.drop_update_tables(conn)
|
||||
freeze.drop_flatnode_file(args.config.FLATNODE_FILE)
|
||||
conn.close()
|
||||
|
||||
return 0
|
||||
@ -82,7 +82,8 @@ class UpdateRefresh:
|
||||
run_legacy_script('update.php', '--recompute-importance',
|
||||
nominatim_env=args, throw_on_fail=True)
|
||||
if args.website:
|
||||
run_legacy_script('setup.php', '--setup-website',
|
||||
nominatim_env=args, throw_on_fail=True)
|
||||
webdir = args.project_dir / 'website'
|
||||
LOG.warning('Setting up website directory at %s', webdir)
|
||||
refresh.setup_website(webdir, args.phplib_dir, args.config)
|
||||
|
||||
return 0
|
||||
|
||||
@ -17,7 +17,7 @@ class Configuration:
|
||||
Nominatim uses dotenv to configure the software. Configuration options
|
||||
are resolved in the following order:
|
||||
|
||||
* from the OS environment
|
||||
* from the OS environment (or the dirctionary given in `environ`
|
||||
* from the .env file in the project directory of the installation
|
||||
* from the default installation in the configuration directory
|
||||
|
||||
@ -25,7 +25,8 @@ class Configuration:
|
||||
avoid conflicts with other environment variables.
|
||||
"""
|
||||
|
||||
def __init__(self, project_dir, config_dir):
|
||||
def __init__(self, project_dir, config_dir, environ=None):
|
||||
self.environ = environ or os.environ
|
||||
self.project_dir = project_dir
|
||||
self.config_dir = config_dir
|
||||
self._config = dotenv_values(str((config_dir / 'env.defaults').resolve()))
|
||||
@ -42,7 +43,7 @@ class Configuration:
|
||||
def __getattr__(self, name):
|
||||
name = 'NOMINATIM_' + name
|
||||
|
||||
return os.environ.get(name) or self._config[name]
|
||||
return self.environ.get(name) or self._config[name]
|
||||
|
||||
def get_bool(self, name):
|
||||
""" Return the given configuration parameter as a boolean.
|
||||
@ -100,6 +101,6 @@ class Configuration:
|
||||
merged in.
|
||||
"""
|
||||
env = dict(self._config)
|
||||
env.update(os.environ)
|
||||
env.update(self.environ)
|
||||
|
||||
return env
|
||||
|
||||
@ -7,6 +7,8 @@ import psycopg2
|
||||
import psycopg2.extensions
|
||||
import psycopg2.extras
|
||||
|
||||
from ..errors import UsageError
|
||||
|
||||
class _Cursor(psycopg2.extras.DictCursor):
|
||||
""" A cursor returning dict-like objects and providing specialised
|
||||
execution functions.
|
||||
@ -42,14 +44,34 @@ class _Connection(psycopg2.extensions.connection):
|
||||
"""
|
||||
return super().cursor(cursor_factory=cursor_factory, **kwargs)
|
||||
|
||||
|
||||
def table_exists(self, table):
|
||||
""" Check that a table with the given name exists in the database.
|
||||
"""
|
||||
with self.cursor() as cur:
|
||||
num = cur.scalar("""SELECT count(*) FROM pg_tables
|
||||
WHERE tablename = %s""", (table, ))
|
||||
WHERE tablename = %s and schemaname = 'public'""", (table, ))
|
||||
return num == 1
|
||||
|
||||
|
||||
def index_exists(self, index, table=None):
|
||||
""" Check that an index with the given name exists in the database.
|
||||
If table is not None then the index must relate to the given
|
||||
table.
|
||||
"""
|
||||
with self.cursor() as cur:
|
||||
cur.execute("""SELECT tablename FROM pg_indexes
|
||||
WHERE indexname = %s and schemaname = 'public'""", (index, ))
|
||||
if cur.rowcount == 0:
|
||||
return False
|
||||
|
||||
if table is not None:
|
||||
row = cur.fetchone()
|
||||
return row[0] == table
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def server_version_tuple(self):
|
||||
""" Return the server version as a tuple of (major, minor).
|
||||
Converts correctly for pre-10 and post-10 PostgreSQL versions.
|
||||
@ -64,4 +86,7 @@ def connect(dsn):
|
||||
""" Open a connection to the database using the specialised connection
|
||||
factory.
|
||||
"""
|
||||
return psycopg2.connect(dsn, connection_factory=_Connection)
|
||||
try:
|
||||
return psycopg2.connect(dsn, connection_factory=_Connection)
|
||||
except psycopg2.OperationalError as err:
|
||||
raise UsageError("Cannot connect to database: {}".format(err)) from err
|
||||
|
||||
269
nominatim/tools/check_database.py
Normal file
269
nominatim/tools/check_database.py
Normal file
@ -0,0 +1,269 @@
|
||||
"""
|
||||
Collection of functions that check if the database is complete and functional.
|
||||
"""
|
||||
from enum import Enum
|
||||
from textwrap import dedent
|
||||
|
||||
import psycopg2
|
||||
|
||||
from ..db.connection import connect
|
||||
from ..errors import UsageError
|
||||
|
||||
CHECKLIST = []
|
||||
|
||||
class CheckState(Enum):
|
||||
""" Possible states of a check. FATAL stops check execution entirely.
|
||||
"""
|
||||
OK = 0
|
||||
FAIL = 1
|
||||
FATAL = 2
|
||||
NOT_APPLICABLE = 3
|
||||
|
||||
def _check(hint=None):
|
||||
""" Decorator for checks. It adds the function to the list of
|
||||
checks to execute and adds the code for printing progress messages.
|
||||
"""
|
||||
def decorator(func):
|
||||
title = func.__doc__.split('\n', 1)[0].strip()
|
||||
def run_check(conn, config):
|
||||
print(title, end=' ... ')
|
||||
ret = func(conn, config)
|
||||
if isinstance(ret, tuple):
|
||||
ret, params = ret
|
||||
else:
|
||||
params = {}
|
||||
if ret == CheckState.OK:
|
||||
print('\033[92mOK\033[0m')
|
||||
elif ret == CheckState.NOT_APPLICABLE:
|
||||
print('not applicable')
|
||||
else:
|
||||
print('\x1B[31mFailed\033[0m')
|
||||
if hint:
|
||||
print(dedent(hint.format(**params)))
|
||||
return ret
|
||||
|
||||
CHECKLIST.append(run_check)
|
||||
return run_check
|
||||
|
||||
return decorator
|
||||
|
||||
class _BadConnection: # pylint: disable=R0903
|
||||
|
||||
def __init__(self, msg):
|
||||
self.msg = msg
|
||||
|
||||
def close(self):
|
||||
""" Dummy function to provide the implementation.
|
||||
"""
|
||||
|
||||
def check_database(config):
|
||||
""" Run a number of checks on the database and return the status.
|
||||
"""
|
||||
try:
|
||||
conn = connect(config.get_libpq_dsn())
|
||||
except UsageError as err:
|
||||
conn = _BadConnection(str(err))
|
||||
|
||||
overall_result = 0
|
||||
for check in CHECKLIST:
|
||||
ret = check(conn, config)
|
||||
if ret == CheckState.FATAL:
|
||||
conn.close()
|
||||
return 1
|
||||
if ret in (CheckState.FATAL, CheckState.FAIL):
|
||||
overall_result = 1
|
||||
|
||||
conn.close()
|
||||
return overall_result
|
||||
|
||||
|
||||
def _get_indexes(conn):
|
||||
indexes = ['idx_word_word_id',
|
||||
'idx_place_addressline_address_place_id',
|
||||
'idx_placex_rank_search',
|
||||
'idx_placex_rank_address',
|
||||
'idx_placex_parent_place_id',
|
||||
'idx_placex_geometry_reverse_lookuppolygon',
|
||||
'idx_placex_geometry_reverse_placenode',
|
||||
'idx_osmline_parent_place_id',
|
||||
'idx_osmline_parent_osm_id',
|
||||
'idx_postcode_id',
|
||||
'idx_postcode_postcode'
|
||||
]
|
||||
if conn.table_exists('search_name'):
|
||||
indexes.extend(('idx_search_name_nameaddress_vector',
|
||||
'idx_search_name_name_vector',
|
||||
'idx_search_name_centroid'))
|
||||
if conn.table_exists('place'):
|
||||
indexes.extend(('idx_placex_pendingsector',
|
||||
'idx_location_area_country_place_id',
|
||||
'idx_place_osm_unique'
|
||||
))
|
||||
|
||||
return indexes
|
||||
|
||||
|
||||
### CHECK FUNCTIONS
|
||||
#
|
||||
# Functions are exectured in the order they appear here.
|
||||
|
||||
@_check(hint="""\
|
||||
{error}
|
||||
|
||||
Hints:
|
||||
* Is the database server started?
|
||||
* Check the NOMINATIM_DATABASE_DSN variable in your local .env
|
||||
* Try connecting to the database with the same settings
|
||||
|
||||
Project directory: {config.project_dir}
|
||||
Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
|
||||
""")
|
||||
def check_connection(conn, config):
|
||||
""" Checking database connection
|
||||
"""
|
||||
if isinstance(conn, _BadConnection):
|
||||
return CheckState.FATAL, dict(error=conn.msg, config=config)
|
||||
|
||||
return CheckState.OK
|
||||
|
||||
@_check(hint="""\
|
||||
placex table not found
|
||||
|
||||
Hints:
|
||||
* Are you connecting to the right database?
|
||||
* Did the import process finish without errors?
|
||||
|
||||
Project directory: {config.project_dir}
|
||||
Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN}
|
||||
""")
|
||||
def check_placex_table(conn, config):
|
||||
""" Checking for placex table
|
||||
"""
|
||||
if conn.table_exists('placex'):
|
||||
return CheckState.OK
|
||||
|
||||
return CheckState.FATAL, dict(config=config)
|
||||
|
||||
|
||||
@_check(hint="""placex table has no data. Did the import finish sucessfully?""")
|
||||
def check_placex_size(conn, config): # pylint: disable=W0613
|
||||
""" Checking for placex content
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x')
|
||||
|
||||
return CheckState.OK if cnt > 0 else CheckState.FATAL
|
||||
|
||||
|
||||
@_check(hint="""\
|
||||
The Postgresql extension nominatim.so was not correctly loaded.
|
||||
|
||||
Error: {error}
|
||||
|
||||
Hints:
|
||||
* Check the output of the CMmake/make installation step
|
||||
* Does nominatim.so exist?
|
||||
* Does nominatim.so exist on the database server?
|
||||
* Can nominatim.so be accessed by the database user?
|
||||
""")
|
||||
def check_module(conn, config): # pylint: disable=W0613
|
||||
""" Checking that nominatim.so module is installed
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
try:
|
||||
out = cur.scalar("SELECT make_standard_name('a')")
|
||||
except psycopg2.ProgrammingError as err:
|
||||
return CheckState.FAIL, dict(error=str(err))
|
||||
|
||||
if out != 'a':
|
||||
return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()')
|
||||
|
||||
return CheckState.OK
|
||||
|
||||
|
||||
@_check(hint="""\
|
||||
The indexing didn't finish. {count} entries are not yet indexed.
|
||||
|
||||
To index the remaining entries, run: {index_cmd}
|
||||
""")
|
||||
def check_indexing(conn, config): # pylint: disable=W0613
|
||||
""" Checking indexing status
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0')
|
||||
|
||||
if cnt == 0:
|
||||
return CheckState.OK
|
||||
|
||||
if conn.index_exists('idx_word_word_id'):
|
||||
# Likely just an interrupted update.
|
||||
index_cmd = 'nominatim index'
|
||||
else:
|
||||
# Looks like the import process got interrupted.
|
||||
index_cmd = 'nominatim import --continue indexing'
|
||||
|
||||
return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd)
|
||||
|
||||
|
||||
@_check(hint="""\
|
||||
The following indexes are missing:
|
||||
{indexes}
|
||||
|
||||
Rerun the index creation with: nominatim import --continue db-postprocess
|
||||
""")
|
||||
def check_database_indexes(conn, config): # pylint: disable=W0613
|
||||
""" Checking that database indexes are complete
|
||||
"""
|
||||
missing = []
|
||||
for index in _get_indexes(conn):
|
||||
if not conn.index_exists(index):
|
||||
missing.append(index)
|
||||
|
||||
if missing:
|
||||
return CheckState.FAIL, dict(indexes='\n '.join(missing))
|
||||
|
||||
return CheckState.OK
|
||||
|
||||
|
||||
@_check(hint="""\
|
||||
At least one index is invalid. That can happen, e.g. when index creation was
|
||||
disrupted and later restarted. You should delete the affected indices
|
||||
and recreate them.
|
||||
|
||||
Invalid indexes:
|
||||
{indexes}
|
||||
""")
|
||||
def check_database_index_valid(conn, config): # pylint: disable=W0613
|
||||
""" Checking that all database indexes are valid
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(""" SELECT relname FROM pg_class, pg_index
|
||||
WHERE pg_index.indisvalid = false
|
||||
AND pg_index.indexrelid = pg_class.oid""")
|
||||
|
||||
broken = list(cur)
|
||||
|
||||
if broken:
|
||||
return CheckState.FAIL, dict(indexes='\n '.join(broken))
|
||||
|
||||
return CheckState.OK
|
||||
|
||||
|
||||
@_check(hint="""\
|
||||
{error}
|
||||
Run TIGER import again: nominatim add-data --tiger-data <DIR>
|
||||
""")
|
||||
def check_tiger_table(conn, config):
|
||||
""" Checking TIGER external data table.
|
||||
"""
|
||||
if not config.get_bool('USE_US_TIGER_DATA'):
|
||||
return CheckState.NOT_APPLICABLE
|
||||
|
||||
if not conn.table_exists('location_property_tiger'):
|
||||
return CheckState.FAIL, dict(error='TIGER data table not found.')
|
||||
|
||||
with conn.cursor() as cur:
|
||||
if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0:
|
||||
return CheckState.FAIL, dict(error='TIGER data table is empty.')
|
||||
|
||||
return CheckState.OK
|
||||
43
nominatim/tools/freeze.py
Normal file
43
nominatim/tools/freeze.py
Normal file
@ -0,0 +1,43 @@
|
||||
"""
|
||||
Functions for removing unnecessary data from the database.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
UPDATE_TABLES = [
|
||||
'address_levels',
|
||||
'gb_postcode',
|
||||
'import_osmosis_log',
|
||||
'import_polygon_%',
|
||||
'location_area%',
|
||||
'location_road%',
|
||||
'place',
|
||||
'planet_osm_%',
|
||||
'search_name_%',
|
||||
'us_postcode',
|
||||
'wikipedia_%'
|
||||
]
|
||||
|
||||
def drop_update_tables(conn):
|
||||
""" Drop all tables only necessary for updating the database from
|
||||
OSM replication data.
|
||||
"""
|
||||
|
||||
where = ' or '.join(["(tablename LIKE '{}')".format(t) for t in UPDATE_TABLES])
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT tablename FROM pg_tables WHERE " + where)
|
||||
tables = [r[0] for r in cur]
|
||||
|
||||
for table in tables:
|
||||
cur.execute('DROP TABLE IF EXISTS "{}" CASCADE'.format(table))
|
||||
|
||||
conn.commit()
|
||||
|
||||
|
||||
def drop_flatnode_file(fname):
|
||||
""" Remove the flatnode file if it exists.
|
||||
"""
|
||||
if fname:
|
||||
fpath = Path(fname)
|
||||
if fpath.exists():
|
||||
fpath.unlink()
|
||||
@ -2,12 +2,16 @@
|
||||
Functions for bringing auxiliary data in the database up-to-date.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from textwrap import dedent
|
||||
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
from ..db.utils import execute_file
|
||||
|
||||
LOG = logging.getLogger()
|
||||
|
||||
def update_postcodes(conn, sql_dir):
|
||||
""" Recalculate postcode centroids and add, remove and update entries in the
|
||||
location_postcode table. `conn` is an opne connection to the database.
|
||||
@ -165,3 +169,65 @@ def create_functions(conn, config, sql_dir,
|
||||
cur.execute(sql)
|
||||
|
||||
conn.commit()
|
||||
|
||||
|
||||
WEBSITE_SCRIPTS = (
|
||||
'deletable.php',
|
||||
'details.php',
|
||||
'lookup.php',
|
||||
'polygons.php',
|
||||
'reverse.php',
|
||||
'search.php',
|
||||
'status.php'
|
||||
)
|
||||
|
||||
# constants needed by PHP scripts: PHP name, config name, type
|
||||
PHP_CONST_DEFS = (
|
||||
('Database_DSN', 'DATABASE_DSN', str),
|
||||
('Default_Language', 'DEFAULT_LANGUAGE', str),
|
||||
('Log_DB', 'LOG_DB', bool),
|
||||
('Log_File', 'LOG_FILE', str),
|
||||
('Max_Word_Frequency', 'MAX_WORD_FREQUENCY', int),
|
||||
('NoAccessControl', 'CORS_NOACCESSCONTROL', bool),
|
||||
('Places_Max_ID_count', 'LOOKUP_MAX_COUNT', int),
|
||||
('PolygonOutput_MaximumTypes', 'POLYGON_OUTPUT_MAX_TYPES', int),
|
||||
('Search_BatchMode', 'SEARCH_BATCH_MODE', bool),
|
||||
('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str),
|
||||
('Term_Normalization_Rules', 'TERM_NORMALIZATION', str),
|
||||
('Use_Aux_Location_data', 'USE_AUX_LOCATION_DATA', bool),
|
||||
('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool),
|
||||
('MapIcon_URL', 'MAPICON_URL', str),
|
||||
)
|
||||
|
||||
|
||||
def setup_website(basedir, phplib_dir, config):
|
||||
""" Create the website script stubs.
|
||||
"""
|
||||
if not basedir.exists():
|
||||
LOG.info('Creating website directory.')
|
||||
basedir.mkdir()
|
||||
|
||||
template = dedent("""\
|
||||
<?php
|
||||
|
||||
@define('CONST_Debug', $_GET['debug'] ?? false);
|
||||
@define('CONST_LibDir', '{}');
|
||||
|
||||
""".format(phplib_dir))
|
||||
|
||||
for php_name, conf_name, var_type in PHP_CONST_DEFS:
|
||||
if var_type == bool:
|
||||
varout = 'true' if config.get_bool(conf_name) else 'false'
|
||||
elif var_type == int:
|
||||
varout = getattr(config, conf_name)
|
||||
elif not getattr(config, conf_name):
|
||||
varout = 'false'
|
||||
else:
|
||||
varout = "'{}'".format(getattr(config, conf_name).replace("'", "\\'"))
|
||||
|
||||
template += "@define('CONST_{}', {});\n".format(php_name, varout)
|
||||
|
||||
template += "\nrequire_once('{}/website/{{}}');\n".format(phplib_dir)
|
||||
|
||||
for script in WEBSITE_SCRIPTS:
|
||||
(basedir / script).write_text(template.format(script), 'utf-8')
|
||||
|
||||
@ -8,6 +8,7 @@ import psycopg2.extras
|
||||
sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..').resolve()))
|
||||
|
||||
from nominatim.config import Configuration
|
||||
from nominatim.tools import refresh
|
||||
from steps.utils import run_script
|
||||
|
||||
class NominatimEnvironment:
|
||||
@ -104,7 +105,8 @@ class NominatimEnvironment:
|
||||
self.website_dir.cleanup()
|
||||
|
||||
self.website_dir = tempfile.TemporaryDirectory()
|
||||
self.run_setup_script('setup-website')
|
||||
cfg = Configuration(None, self.src_dir / 'settings', environ=self.test_env)
|
||||
refresh.setup_website(Path(self.website_dir.name) / 'website', self.src_dir / 'lib-php', cfg)
|
||||
|
||||
|
||||
def db_drop_database(self, name):
|
||||
@ -182,6 +184,7 @@ class NominatimEnvironment:
|
||||
try:
|
||||
self.run_setup_script('all', osm_file=self.api_test_file)
|
||||
self.run_setup_script('import-tiger-data')
|
||||
self.run_setup_script('drop')
|
||||
|
||||
phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
|
||||
run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
|
||||
|
||||
@ -159,31 +159,11 @@ class DBTest extends \PHPUnit\Framework\TestCase
|
||||
|
||||
# Tables, Indices
|
||||
{
|
||||
$this->assertEmpty($oDB->getListOfTables());
|
||||
$oDB->exec('CREATE TABLE table1 (id integer, city varchar, country varchar)');
|
||||
$oDB->exec('CREATE TABLE table2 (id integer, city varchar, country varchar)');
|
||||
$this->assertEquals(
|
||||
array('table1', 'table2'),
|
||||
$oDB->getListOfTables()
|
||||
);
|
||||
$this->assertTrue($oDB->deleteTable('table2'));
|
||||
$this->assertTrue($oDB->deleteTable('table99'));
|
||||
$this->assertEquals(
|
||||
array('table1'),
|
||||
$oDB->getListOfTables()
|
||||
);
|
||||
|
||||
$this->assertTrue($oDB->tableExists('table1'));
|
||||
$this->assertFalse($oDB->tableExists('table99'));
|
||||
$this->assertFalse($oDB->tableExists(null));
|
||||
|
||||
$this->assertEmpty($oDB->getListOfIndices());
|
||||
$oDB->exec('CREATE UNIQUE INDEX table1_index ON table1 (id)');
|
||||
$this->assertEquals(
|
||||
array('table1_index'),
|
||||
$oDB->getListOfIndices()
|
||||
);
|
||||
$this->assertEmpty($oDB->getListOfIndices('table2'));
|
||||
}
|
||||
|
||||
# select queries
|
||||
|
||||
@ -15,26 +15,6 @@ class LibTest extends \PHPUnit\Framework\TestCase
|
||||
$this->assertSame("''", addQuotes(''));
|
||||
}
|
||||
|
||||
|
||||
public function testCreatePointsAroundCenter()
|
||||
{
|
||||
// you might say we're creating a circle
|
||||
$aPoints = createPointsAroundCenter(0, 0, 2);
|
||||
|
||||
$this->assertEquals(
|
||||
101,
|
||||
count($aPoints)
|
||||
);
|
||||
$this->assertEquals(
|
||||
array(
|
||||
array('', 0, 2),
|
||||
array('', 0.12558103905863, 1.9960534568565),
|
||||
array('', 0.25066646712861, 1.984229402629)
|
||||
),
|
||||
array_splice($aPoints, 0, 3)
|
||||
);
|
||||
}
|
||||
|
||||
public function testParseLatLon()
|
||||
{
|
||||
// no coordinates expected
|
||||
@ -132,12 +112,4 @@ class LibTest extends \PHPUnit\Framework\TestCase
|
||||
// start == end
|
||||
$this->closestHouseNumberEvenOddOther(50, 50, 0.5, array('even' => 50, 'odd' => 50, 'other' => 50));
|
||||
}
|
||||
|
||||
public function testGetSearchRankLabel()
|
||||
{
|
||||
$this->assertEquals('unknown', getSearchRankLabel(null));
|
||||
$this->assertEquals('continent', getSearchRankLabel(0));
|
||||
$this->assertEquals('continent', getSearchRankLabel(1));
|
||||
$this->assertEquals('other: 30', getSearchRankLabel(30));
|
||||
}
|
||||
}
|
||||
|
||||
@ -36,6 +36,14 @@ class _TestingCursor(psycopg2.extras.DictCursor):
|
||||
|
||||
return set((tuple(row) for row in self))
|
||||
|
||||
def table_exists(self, table):
|
||||
""" Check that a table with the given name exists in the database.
|
||||
"""
|
||||
num = self.scalar("""SELECT count(*) FROM pg_tables
|
||||
WHERE tablename = %s""", (table, ))
|
||||
return num == 1
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_db(monkeypatch):
|
||||
""" Create an empty database for the test. The database name is also
|
||||
|
||||
@ -15,6 +15,9 @@ import nominatim.clicmd.api
|
||||
import nominatim.clicmd.refresh
|
||||
import nominatim.clicmd.admin
|
||||
import nominatim.indexer.indexer
|
||||
import nominatim.tools.admin
|
||||
import nominatim.tools.check_database
|
||||
import nominatim.tools.freeze
|
||||
import nominatim.tools.refresh
|
||||
import nominatim.tools.replication
|
||||
from nominatim.errors import UsageError
|
||||
@ -50,6 +53,14 @@ def mock_run_legacy(monkeypatch):
|
||||
monkeypatch.setattr(nominatim.cli, 'run_legacy_script', mock)
|
||||
return mock
|
||||
|
||||
@pytest.fixture
|
||||
def mock_func_factory(monkeypatch):
|
||||
def get_mock(module, func):
|
||||
mock = MockParamCapture()
|
||||
monkeypatch.setattr(module, func, mock)
|
||||
return mock
|
||||
|
||||
return get_mock
|
||||
|
||||
def test_cli_help(capsys):
|
||||
""" Running nominatim tool without arguments prints help.
|
||||
@ -62,7 +73,6 @@ def test_cli_help(capsys):
|
||||
|
||||
@pytest.mark.parametrize("command,script", [
|
||||
(('import', '--continue', 'load-data'), 'setup'),
|
||||
(('freeze',), 'setup'),
|
||||
(('special-phrases',), 'specialphrases'),
|
||||
(('add-data', '--tiger-data', 'tiger'), 'setup'),
|
||||
(('add-data', '--file', 'foo.osm'), 'update'),
|
||||
@ -75,26 +85,42 @@ def test_legacy_commands_simple(mock_run_legacy, command, script):
|
||||
assert mock_run_legacy.last_args[0] == script + '.php'
|
||||
|
||||
|
||||
def test_freeze_command(mock_func_factory, temp_db):
|
||||
mock_drop = mock_func_factory(nominatim.tools.freeze, 'drop_update_tables')
|
||||
mock_flatnode = mock_func_factory(nominatim.tools.freeze, 'drop_flatnode_file')
|
||||
|
||||
assert 0 == call_nominatim('freeze')
|
||||
|
||||
assert mock_drop.called == 1
|
||||
assert mock_flatnode.called == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("params", [('--warm', ),
|
||||
('--warm', '--reverse-only'),
|
||||
('--warm', '--search-only'),
|
||||
('--check-database', )])
|
||||
def test_admin_command_legacy(monkeypatch, params):
|
||||
mock_run_legacy = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.clicmd.admin, 'run_legacy_script', mock_run_legacy)
|
||||
('--warm', '--search-only')])
|
||||
def test_admin_command_legacy(mock_func_factory, params):
|
||||
mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script')
|
||||
|
||||
assert 0 == call_nominatim('admin', *params)
|
||||
|
||||
assert mock_run_legacy.called == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))])
|
||||
def test_admin_command_tool(temp_db, monkeypatch, func, params):
|
||||
mock = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.tools.admin, func, mock)
|
||||
def test_admin_command_tool(temp_db, mock_func_factory, func, params):
|
||||
mock = mock_func_factory(nominatim.tools.admin, func)
|
||||
|
||||
assert 0 == call_nominatim('admin', *params)
|
||||
assert mock.called == 1
|
||||
|
||||
|
||||
def test_admin_command_check_database(mock_func_factory):
|
||||
mock = mock_func_factory(nominatim.tools.check_database, 'check_database')
|
||||
|
||||
assert 0 == call_nominatim('admin', '--check-database')
|
||||
assert mock.called == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc'),
|
||||
('node', 12), ('way', 8), ('relation', 32)])
|
||||
def test_add_data_command(mock_run_legacy, name, oid):
|
||||
@ -109,12 +135,10 @@ def test_add_data_command(mock_run_legacy, name, oid):
|
||||
(['--boundaries-only'], 1, 0),
|
||||
(['--no-boundaries'], 0, 1),
|
||||
(['--boundaries-only', '--no-boundaries'], 0, 0)])
|
||||
def test_index_command(monkeypatch, temp_db_cursor, params, do_bnds, do_ranks):
|
||||
def test_index_command(mock_func_factory, temp_db_cursor, params, do_bnds, do_ranks):
|
||||
temp_db_cursor.execute("CREATE TABLE import_status (indexed bool)")
|
||||
bnd_mock = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', bnd_mock)
|
||||
rank_mock = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', rank_mock)
|
||||
bnd_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_boundaries')
|
||||
rank_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_by_rank')
|
||||
|
||||
assert 0 == call_nominatim('index', *params)
|
||||
|
||||
@ -125,11 +149,9 @@ def test_index_command(monkeypatch, temp_db_cursor, params, do_bnds, do_ranks):
|
||||
@pytest.mark.parametrize("command,params", [
|
||||
('wiki-data', ('setup.php', '--import-wikipedia-articles')),
|
||||
('importance', ('update.php', '--recompute-importance')),
|
||||
('website', ('setup.php', '--setup-website')),
|
||||
])
|
||||
def test_refresh_legacy_command(monkeypatch, temp_db, command, params):
|
||||
mock_run_legacy = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.clicmd.refresh, 'run_legacy_script', mock_run_legacy)
|
||||
def test_refresh_legacy_command(mock_func_factory, temp_db, command, params):
|
||||
mock_run_legacy = mock_func_factory(nominatim.clicmd.refresh, 'run_legacy_script')
|
||||
|
||||
assert 0 == call_nominatim('refresh', '--' + command)
|
||||
|
||||
@ -142,18 +164,17 @@ def test_refresh_legacy_command(monkeypatch, temp_db, command, params):
|
||||
('word-counts', 'recompute_word_counts'),
|
||||
('address-levels', 'load_address_levels_from_file'),
|
||||
('functions', 'create_functions'),
|
||||
('website', 'setup_website'),
|
||||
])
|
||||
def test_refresh_command(monkeypatch, temp_db, command, func):
|
||||
func_mock = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.tools.refresh, func, func_mock)
|
||||
def test_refresh_command(mock_func_factory, temp_db, command, func):
|
||||
func_mock = mock_func_factory(nominatim.tools.refresh, func)
|
||||
|
||||
assert 0 == call_nominatim('refresh', '--' + command)
|
||||
assert func_mock.called == 1
|
||||
|
||||
|
||||
def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db):
|
||||
mock_run_legacy = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.clicmd.refresh, 'run_legacy_script', mock_run_legacy)
|
||||
def test_refresh_importance_computed_after_wiki_import(mock_func_factory, temp_db):
|
||||
mock_run_legacy = mock_func_factory(nominatim.clicmd.refresh, 'run_legacy_script')
|
||||
|
||||
assert 0 == call_nominatim('refresh', '--importance', '--wiki-data')
|
||||
|
||||
@ -165,9 +186,8 @@ def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db):
|
||||
(('--init', '--no-update-functions'), 'init_replication'),
|
||||
(('--check-for-updates',), 'check_for_updates')
|
||||
])
|
||||
def test_replication_command(monkeypatch, temp_db, params, func):
|
||||
func_mock = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.tools.replication, func, func_mock)
|
||||
def test_replication_command(mock_func_factory, temp_db, params, func):
|
||||
func_mock = mock_func_factory(nominatim.tools.replication, func)
|
||||
|
||||
assert 0 == call_nominatim('replication', *params)
|
||||
assert func_mock.called == 1
|
||||
@ -188,11 +208,10 @@ def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db):
|
||||
|
||||
@pytest.mark.parametrize("state", [nominatim.tools.replication.UpdateState.UP_TO_DATE,
|
||||
nominatim.tools.replication.UpdateState.NO_CHANGES])
|
||||
def test_replication_update_once_no_index(monkeypatch, temp_db, temp_db_conn,
|
||||
def test_replication_update_once_no_index(mock_func_factory, temp_db, temp_db_conn,
|
||||
status_table, state):
|
||||
status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1)
|
||||
func_mock = MockParamCapture(retval=state)
|
||||
monkeypatch.setattr(nominatim.tools.replication, 'update', func_mock)
|
||||
func_mock = mock_func_factory(nominatim.tools.replication, 'update')
|
||||
|
||||
assert 0 == call_nominatim('replication', '--once', '--no-index')
|
||||
|
||||
@ -236,9 +255,8 @@ def test_replication_update_continuous_no_change(monkeypatch, temp_db_conn, stat
|
||||
assert sleep_mock.last_args[0] == 60
|
||||
|
||||
|
||||
def test_serve_command(monkeypatch):
|
||||
func = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.cli, 'run_php_server', func)
|
||||
def test_serve_command(mock_func_factory):
|
||||
func = mock_func_factory(nominatim.cli, 'run_php_server')
|
||||
|
||||
call_nominatim('serve')
|
||||
|
||||
@ -254,9 +272,8 @@ def test_serve_command(monkeypatch):
|
||||
('details', '--place_id', '10001'),
|
||||
('status',)
|
||||
])
|
||||
def test_api_commands_simple(monkeypatch, params):
|
||||
mock_run_api = MockParamCapture()
|
||||
monkeypatch.setattr(nominatim.clicmd.api, 'run_api_script', mock_run_api)
|
||||
def test_api_commands_simple(mock_func_factory, params):
|
||||
mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script')
|
||||
|
||||
assert 0 == call_nominatim(*params)
|
||||
|
||||
|
||||
@ -20,12 +20,31 @@ def test_connection_table_exists(db, temp_db_cursor):
|
||||
assert db.table_exists('foobar') == True
|
||||
|
||||
|
||||
def test_connection_index_exists(db, temp_db_cursor):
|
||||
assert db.index_exists('some_index') == False
|
||||
|
||||
temp_db_cursor.execute('CREATE TABLE foobar (id INT)')
|
||||
temp_db_cursor.execute('CREATE INDEX some_index ON foobar(id)')
|
||||
|
||||
assert db.index_exists('some_index') == True
|
||||
assert db.index_exists('some_index', table='foobar') == True
|
||||
assert db.index_exists('some_index', table='bar') == False
|
||||
|
||||
|
||||
def test_connection_server_version_tuple(db):
|
||||
ver = db.server_version_tuple()
|
||||
|
||||
assert isinstance(ver, tuple)
|
||||
assert len(ver) == 2
|
||||
assert ver[0] > 8
|
||||
|
||||
def test_cursor_scalar(db, temp_db_cursor):
|
||||
temp_db_cursor.execute('CREATE TABLE dummy (id INT)')
|
||||
|
||||
with db.cursor() as cur:
|
||||
assert cur.scalar('SELECT count(*) FROM dummy') == 0
|
||||
|
||||
|
||||
def test_cursor_scalar_many_rows(db):
|
||||
with db.cursor() as cur:
|
||||
with pytest.raises(RuntimeError):
|
||||
|
||||
76
test/python/test_tools_check_database.py
Normal file
76
test/python/test_tools_check_database.py
Normal file
@ -0,0 +1,76 @@
|
||||
"""
|
||||
Tests for database integrity checks.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from nominatim.tools import check_database as chkdb
|
||||
|
||||
def test_check_database_unknown_db(def_config, monkeypatch):
|
||||
monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'pgsql:dbname=fjgkhughwgh2423gsags')
|
||||
assert 1 == chkdb.check_database(def_config)
|
||||
|
||||
|
||||
def test_check_conection_good(temp_db_conn, def_config):
|
||||
assert chkdb.check_connection(temp_db_conn, def_config) == chkdb.CheckState.OK
|
||||
|
||||
|
||||
def test_check_conection_bad(def_config):
|
||||
badconn = chkdb._BadConnection('Error')
|
||||
assert chkdb.check_connection(badconn, def_config) == chkdb.CheckState.FATAL
|
||||
|
||||
|
||||
def test_check_placex_table_good(temp_db_cursor, temp_db_conn, def_config):
|
||||
temp_db_cursor.execute('CREATE TABLE placex (place_id int)')
|
||||
assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.OK
|
||||
|
||||
|
||||
def test_check_placex_table_bad(temp_db_conn, def_config):
|
||||
assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.FATAL
|
||||
|
||||
|
||||
def test_check_placex_table_size_good(temp_db_cursor, temp_db_conn, def_config):
|
||||
temp_db_cursor.execute('CREATE TABLE placex (place_id int)')
|
||||
temp_db_cursor.execute('INSERT INTO placex VALUES (1), (2)')
|
||||
assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.OK
|
||||
|
||||
|
||||
def test_check_placex_table_size_bad(temp_db_cursor, temp_db_conn, def_config):
|
||||
temp_db_cursor.execute('CREATE TABLE placex (place_id int)')
|
||||
assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.FATAL
|
||||
|
||||
|
||||
def test_check_module_bad(temp_db_conn, def_config):
|
||||
assert chkdb.check_module(temp_db_conn, def_config) == chkdb.CheckState.FAIL
|
||||
|
||||
|
||||
def test_check_indexing_good(temp_db_cursor, temp_db_conn, def_config):
|
||||
temp_db_cursor.execute('CREATE TABLE placex (place_id int, indexed_status smallint)')
|
||||
temp_db_cursor.execute('INSERT INTO placex VALUES (1, 0), (2, 0)')
|
||||
assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.OK
|
||||
|
||||
|
||||
def test_check_indexing_bad(temp_db_cursor, temp_db_conn, def_config):
|
||||
temp_db_cursor.execute('CREATE TABLE placex (place_id int, indexed_status smallint)')
|
||||
temp_db_cursor.execute('INSERT INTO placex VALUES (1, 0), (2, 2)')
|
||||
assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.FAIL
|
||||
|
||||
|
||||
def test_check_database_indexes_bad(temp_db_conn, def_config):
|
||||
assert chkdb.check_database_indexes(temp_db_conn, def_config) == chkdb.CheckState.FAIL
|
||||
|
||||
|
||||
def test_check_tiger_table_disabled(temp_db_conn, def_config, monkeypatch):
|
||||
monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA' , 'no')
|
||||
assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.NOT_APPLICABLE
|
||||
|
||||
|
||||
def test_check_tiger_table_enabled(temp_db_cursor, temp_db_conn, def_config, monkeypatch):
|
||||
monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA' , 'yes')
|
||||
assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.FAIL
|
||||
|
||||
temp_db_cursor.execute('CREATE TABLE location_property_tiger (place_id int)')
|
||||
assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.FAIL
|
||||
|
||||
temp_db_cursor.execute('INSERT INTO location_property_tiger VALUES (1), (2)')
|
||||
assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.OK
|
||||
|
||||
51
test/python/test_tools_freeze.py
Normal file
51
test/python/test_tools_freeze.py
Normal file
@ -0,0 +1,51 @@
|
||||
"""
|
||||
Tests for freeze functions (removing unused database parts).
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from nominatim.tools import freeze
|
||||
|
||||
NOMINATIM_RUNTIME_TABLES = [
|
||||
'country_name', 'country_osm_grid',
|
||||
'location_postcode', 'location_property_osmline', 'location_property_tiger',
|
||||
'placex', 'place_adressline',
|
||||
'search_name',
|
||||
'word'
|
||||
]
|
||||
|
||||
NOMINATIM_DROP_TABLES = [
|
||||
'address_levels',
|
||||
'location_area', 'location_area_country', 'location_area_large_100',
|
||||
'location_road_1',
|
||||
'place', 'planet_osm_nodes', 'planet_osm_rels', 'planet_osm_ways',
|
||||
'search_name_111',
|
||||
'wikipedia_article', 'wikipedia_redirect'
|
||||
]
|
||||
|
||||
def test_drop_tables(temp_db_conn, temp_db_cursor):
|
||||
for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES:
|
||||
temp_db_cursor.execute('CREATE TABLE {} (id int)'.format(table))
|
||||
|
||||
freeze.drop_update_tables(temp_db_conn)
|
||||
|
||||
for table in NOMINATIM_RUNTIME_TABLES:
|
||||
assert temp_db_cursor.table_exists(table)
|
||||
|
||||
for table in NOMINATIM_DROP_TABLES:
|
||||
assert not temp_db_cursor.table_exists(table)
|
||||
|
||||
def test_drop_flatnode_file_no_file():
|
||||
freeze.drop_flatnode_file('')
|
||||
|
||||
|
||||
def test_drop_flatnode_file_file_already_gone(tmp_path):
|
||||
freeze.drop_flatnode_file(str(tmp_path / 'something.store'))
|
||||
|
||||
|
||||
def test_drop_flatnode_file_delte(tmp_path):
|
||||
flatfile = tmp_path / 'flatnode.store'
|
||||
flatfile.write_text('Some content')
|
||||
|
||||
freeze.drop_flatnode_file(str(flatfile))
|
||||
|
||||
assert not flatfile.exists()
|
||||
70
test/python/test_tools_refresh_setup_website.py
Normal file
70
test/python/test_tools_refresh_setup_website.py
Normal file
@ -0,0 +1,70 @@
|
||||
"""
|
||||
Tests for setting up the website scripts.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from nominatim.tools import refresh
|
||||
|
||||
@pytest.fixture
|
||||
def envdir(tmpdir):
|
||||
(tmpdir / 'php').mkdir()
|
||||
(tmpdir / 'php' / 'website').mkdir()
|
||||
return tmpdir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_script(envdir):
|
||||
def _create_file(code):
|
||||
outfile = envdir / 'php' / 'website' / 'search.php'
|
||||
outfile.write_text('<?php\n{}\n'.format(code), 'utf-8')
|
||||
|
||||
return _create_file
|
||||
|
||||
|
||||
def run_website_script(envdir, config):
|
||||
refresh.setup_website(envdir, envdir / 'php', config)
|
||||
|
||||
proc = subprocess.run(['/usr/bin/env', 'php', '-Cq',
|
||||
envdir / 'search.php'], check=False)
|
||||
|
||||
return proc.returncode
|
||||
|
||||
|
||||
@pytest.mark.parametrize("setting,retval", (('yes', 10), ('no', 20)))
|
||||
def test_setup_website_check_bool(def_config, monkeypatch, envdir, test_script,
|
||||
setting, retval):
|
||||
monkeypatch.setenv('NOMINATIM_CORS_NOACCESSCONTROL', setting)
|
||||
|
||||
test_script('exit(CONST_NoAccessControl ? 10 : 20);')
|
||||
|
||||
assert run_website_script(envdir, def_config) == retval
|
||||
|
||||
|
||||
@pytest.mark.parametrize("setting", (0, 10, 99067))
|
||||
def test_setup_website_check_int(def_config, monkeypatch, envdir, test_script, setting):
|
||||
monkeypatch.setenv('NOMINATIM_LOOKUP_MAX_COUNT', str(setting))
|
||||
|
||||
test_script('exit(CONST_Places_Max_ID_count == {} ? 10 : 20);'.format(setting))
|
||||
|
||||
assert run_website_script(envdir, def_config) == 10
|
||||
|
||||
|
||||
def test_setup_website_check_empty_str(def_config, monkeypatch, envdir, test_script):
|
||||
monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', '')
|
||||
|
||||
test_script('exit(CONST_Default_Language === false ? 10 : 20);')
|
||||
|
||||
assert run_website_script(envdir, def_config) == 10
|
||||
|
||||
|
||||
def test_setup_website_check_str(def_config, monkeypatch, envdir, test_script):
|
||||
monkeypatch.setenv('NOMINATIM_DEFAULT_LANGUAGE', 'ffde 2')
|
||||
|
||||
test_script('exit(CONST_Default_Language === "ffde 2" ? 10 : 20);')
|
||||
|
||||
assert run_website_script(envdir, def_config) == 10
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user