osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-11-25 19:35:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2278 from lonvia/remove-transistion-functions

Browse Source 
Remove transition functions
This commit is contained in:
Sarah Hoffmann 2021-04-17 10:13:33 +02:00 committed by GitHub
commit 9fabc5572d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 0 additions and 833 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
CMakeLists.txt
View File

@ -109,21 +109,6 @@ if (BUILD_IMPORTER)
" wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz")
endif()
set(CUSTOMSCRIPTS
check_import_finished.php
country_languages.php
export.php
query.php
setup.php
update.php
warm.php
)
foreach (script_source ${CUSTOMSCRIPTS})
configure_file(${PROJECT_SOURCE_DIR}/cmake/script.tmpl
${PROJECT_BINARY_DIR}/utils/${script_source})
endforeach()
configure_file(${PROJECT_SOURCE_DIR}/cmake/tool.tmpl
${PROJECT_BINARY_DIR}/nominatim)
endif()

14
cmake/script.tmpl
View File

@ -1,14 +0,0 @@
#!@PHP_BIN@ -Cq
<?php
require('@CMAKE_SOURCE_DIR@/lib-php/dotenv_loader.php');
@define('CONST_Default_ModulePath', '@CMAKE_BINARY_DIR@/module');
@define('CONST_Default_Osm2pgsql', '@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql');
@define('CONST_DataDir', '@CMAKE_SOURCE_DIR@/data');
@define('CONST_SqlDir', '@CMAKE_SOURCE_DIR@/lib-sql');
@define('CONST_ConfigDir', '@CMAKE_SOURCE_DIR@/settings');
loadDotEnv();
$_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim';
require_once('@CMAKE_SOURCE_DIR@/lib-php/admin/@script_source@');

10
lib-php/admin/check_import_finished.php
View File

@ -1,10 +0,0 @@
<?php
@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
loadSettings(getcwd());
(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
->addParams('admin', '--check-database')
->run();

218
lib-php/admin/setup.php
View File

@ -1,218 +0,0 @@
<?php
@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
require_once(CONST_LibDir.'/setup/SetupClass.php');
require_once(CONST_LibDir.'/setup_functions.php');
ini_set('memory_limit', '800M');
use Nominatim\Setup\SetupFunctions as SetupFunctions;
// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
$aCMDOptions
= array(
'Create and setup nominatim search system',
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
array('reverse-only', '', 0, 1, 0, 0, 'bool', 'Do not create search tables and indexes'),
array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'),
array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
array('setup-website', '', 0, 1, 0, 0, 'bool', 'Used to compile environment variables for the website'),
array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
);
// $aCMDOptions passed to getCmdOpt by reference
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
loadSettings($aCMDResult['project-dir'] ?? getcwd());
setupHTTPProxy();
$bDidSomething = false;
$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
// by default, use all but one processor, but never more than 15.
$iInstances = max(1, $aCMDResult['threads'] ?? (min(16, getProcessorCount()) - 1));
function run($oCmd)
{
global $iInstances;
global $aCMDResult;
$oCmd->addParams('--threads', $iInstances);
if ($aCMDResult['ignore-errors'] ?? false) {
$oCmd->addParams('--ignore-errors');
}
if ($aCMDResult['quiet'] ?? false) {
$oCmd->addParams('--quiet');
}
if ($aCMDResult['verbose'] ?? false) {
$oCmd->addParams('--verbose');
}
$oCmd->run(true);
}
//*******************************************************
// Making some sanity check:
// Check if osm-file is set and points to a valid file
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
// to remain in /lib/setup_functions.php function
checkInFile($aCMDResult['osm-file']);
}
// ******************************************************
// instantiate Setup class
$oSetup = new SetupFunctions($aCMDResult);
// *******************************************************
// go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
run((clone($oNominatimCmd))->addParams('transition', '--create-db'));
}
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$oCmd = (clone($oNominatimCmd))->addParams('transition', '--setup-db');
if ($aCMDResult['no-partitions'] ?? false) {
$oCmd->addParams('--no-partitions');
}
run($oCmd);
}
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
$bDidSomething = true;
$oCmd = (clone($oNominatimCmd))
->addParams('transition', '--import-data')
->addParams('--osm-file', $aCMDResult['osm-file']);
if ($aCMDResult['drop'] ?? false) {
$oCmd->addParams('--drop');
}
run($oCmd);
}
if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createSqlFunctions();
}
if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
$oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-tables');
if ($aCMDResult['reverse-only'] ?? false) {
$oCmd->addParams('--reverse-only');
}
run($oCmd);
}
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
run((clone($oNominatimCmd))->addParams('transition', '--create-partition-tables'));
}
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createSqlFunctions(); // also create partition functions
}
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
$bDidSomething = true;
// ignore errors!
(clone($oNominatimCmd))->addParams('refresh', '--wiki-data')->run();
}
if ($aCMDResult['load-data'] || $aCMDResult['all']) {
$bDidSomething = true;
run((clone($oNominatimCmd))->addParams('transition', '--load-data'));
}
if ($aCMDResult['import-tiger-data']) {
$bDidSomething = true;
$sTigerPath = getSetting('TIGER_DATA_PATH', CONST_InstallDir.'/tiger');
run((clone($oNominatimCmd))->addParams('transition', '--tiger-data', $sTigerPath));
}
if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->calculatePostcodes($aCMDResult['all']);
}
if ($aCMDResult['index'] || $aCMDResult['all']) {
$bDidSomething = true;
$oCmd = (clone($oNominatimCmd))->addParams('transition', '--index');
if ($aCMDResult['index-noanalyse'] ?? false) {
$oCmd->addParams('--no-analyse');
}
run($oCmd);
}
if ($aCMDResult['drop']) {
$bDidSomething = true;
run((clone($oNominatimCmd))->addParams('freeze'));
}
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
$bDidSomething = true;
$oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-search-indices');
if ($aCMDResult['drop'] ?? false) {
$oCmd->addParams('--drop');
}
run($oCmd);
}
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
$bDidSomething = true;
run(clone($oNominatimCmd))->addParams('transition', '--create-country-names');
}
if ($aCMDResult['setup-website'] || $aCMDResult['all']) {
$bDidSomething = true;
run((clone($oNominatimCmd))->addParams('refresh', '--website'));
}
// ******************************************************
// If we did something, repeat the warnings
if (!$bDidSomething) {
showUsage($aCMDOptions, true);
} else {
echo "Summary of warnings:\n\n";
repeatWarnings();
echo "\n";
info('Setup finished.');
}

11
lib-php/admin/specialphrases.php
View File

@ -1,11 +0,0 @@
<?php
@define('CONST_LibDir', dirname(dirname(__FILE__)));
require_once(CONST_LibDir.'/init-cmd.php');
loadSettings(getcwd());
(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
->addParams('special-phrases', '--import-from-wiki')
->run();

88
lib-php/admin/update.php
View File

@ -3,12 +3,9 @@
require_once(CONST_LibDir.'/init-cmd.php');
require_once(CONST_LibDir.'/setup_functions.php');
require_once(CONST_LibDir.'/setup/SetupClass.php');
ini_set('memory_limit', '800M');
use Nominatim\Setup\SetupFunctions as SetupFunctions;
// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
$aCMDOptions
= array(
@ -17,13 +14,6 @@ $aCMDOptions
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
@ -35,14 +25,6 @@ $aCMDOptions
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
array('update-address-levels', '', 0, 1, 0, 0, 'bool', 'Reimport address level configuration (EXPERT)'),
array('recompute-importance', '', 0, 1, 0, 0, 'bool', 'Recompute place importances'),
array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
);
@ -51,9 +33,6 @@ getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
loadSettings($aCMDResult['project-dir'] ?? getcwd());
setupHTTPProxy();
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
date_default_timezone_set('Etc/UTC');
$oDB = new Nominatim\DB();
@ -103,35 +82,6 @@ if ($fPostgresVersion >= 11.0) {
);
}
$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
function run($oCmd)
{
global $aCMDResult;
if ($aCMDResult['quiet'] ?? false) {
$oCmd->addParams('--quiet');
}
if ($aCMDResult['verbose'] ?? false) {
$oCmd->addParams('--verbose');
}
$oCmd->run(true);
}
if ($aResult['init-updates']) {
$oCmd = (clone($oNominatimCmd))->addParams('replication', '--init');
if ($aResult['no-update-functions']) {
$oCmd->addParams('--no-update-functions');
}
run($oCmd);
}
if ($aResult['check-for-updates']) {
exit((clone($oNominatimCmd))->addParams('replication', '--check-for-updates')->run());
}
if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
// import diffs and files directly (e.g. from osmosis --rri)
$sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
@ -152,10 +102,6 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
// Don't update the import status - we don't know what this file contains
}
if ($aResult['calculate-postcodes']) {
run((clone($oNominatimCmd))->addParams('refresh', '--postcodes'));
}
$sTemporaryFile = CONST_InstallDir.'/osmosischange.osc';
$bHaveDiff = false;
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
@ -200,37 +146,3 @@ if ($bHaveDiff) {
fail("osm2pgsql exited with error level $iRet\n");
}
}
if ($aResult['recompute-word-counts']) {
run((clone($oNominatimCmd))->addParams('refresh', '--word-counts'));
}
if ($aResult['index']) {
run((clone $oNominatimCmd)
->addParams('index', '--minrank', $aResult['index-rank'])
->addParams('--threads', $aResult['index-instances']));
}
if ($aResult['update-address-levels']) {
run((clone($oNominatimCmd))->addParams('refresh', '--address-levels'));
}
if ($aResult['recompute-importance']) {
run((clone($oNominatimCmd))->addParams('refresh', '--importance'));
}
if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
$oCmd = (clone($oNominatimCmd))
->addParams('replication')
->addParams('--threads', $aResult['index-instances']);
if (!$aResult['import-osmosis-all']) {
$oCmd->addParams('--once');
}
if ($aResult['no-index']) {
$oCmd->addParams('--no-index');
}
run($oCmd);
}

52
lib-php/cmd.php
View File

@ -144,58 +144,6 @@ function repeatWarnings()
}
function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreErrors = false)
{
// Convert database DSN to psql parameters
$aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN'));
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
$oCmd = new \Nominatim\Shell('psql');
$oCmd->addParams('--port', $aDSNInfo['port']);
$oCmd->addParams('--dbname', $aDSNInfo['database']);
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
$oCmd->addParams('--host', $aDSNInfo['hostspec']);
}
if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
$oCmd->addParams('--username', $aDSNInfo['username']);
}
if (isset($aDSNInfo['password'])) {
$oCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
}
if (!$bVerbose) {
$oCmd->addParams('--quiet');
}
if ($bfatal && !$bIgnoreErrors) {
$oCmd->addParams('-v', 'ON_ERROR_STOP=1');
}
$aDescriptors = array(
0 => array('pipe', 'r'),
1 => STDOUT,
2 => STDERR
);
$ahPipes = null;
$hProcess = @proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
if (!is_resource($hProcess)) {
fail('unable to start pgsql');
}
if (!$bVerbose) {
fwrite($ahPipes[0], 'set client_min_messages to WARNING;');
}
while (strlen($sScript)) {
$iWritten = fwrite($ahPipes[0], $sScript);
if ($iWritten <= 0) break;
$sScript = substr($sScript, $iWritten);
}
fclose($ahPipes[0]);
$iReturn = proc_close($hProcess);
if ($bfatal && $iReturn > 0) {
fail("pgsql returned with error code ($iReturn)");
}
}
function setupHTTPProxy()
{
if (!getSettingBool('HTTP_PROXY')) {

261
lib-php/setup/SetupClass.php
View File

@ -1,261 +0,0 @@
<?php
namespace Nominatim\Setup;
require_once(CONST_LibDir.'/Shell.php');
class SetupFunctions
{
protected $iInstances;
protected $aDSNInfo;
protected $bQuiet;
protected $bVerbose;
protected $sIgnoreErrors;
protected $bEnableDiffUpdates;
protected $bEnableDebugStatements;
protected $bDrop;
protected $oDB = null;
protected $oNominatimCmd;
public function __construct(array $aCMDResult)
{
// by default, use all but one processor, but never more than 15.
$this->iInstances = isset($aCMDResult['threads'])
? $aCMDResult['threads']
: (min(16, getProcessorCount()) - 1);
if ($this->iInstances < 1) {
$this->iInstances = 1;
warn('resetting threads to '.$this->iInstances);
}
// parse database string
$this->aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN'));
if (!isset($this->aDSNInfo['port'])) {
$this->aDSNInfo['port'] = 5432;
}
// setting member variables based on command line options stored in $aCMDResult
$this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet'];
$this->bVerbose = $aCMDResult['verbose'];
//setting default values which are not set by the update.php array
if (isset($aCMDResult['ignore-errors'])) {
$this->sIgnoreErrors = $aCMDResult['ignore-errors'];
} else {
$this->sIgnoreErrors = false;
}
if (isset($aCMDResult['enable-debug-statements'])) {
$this->bEnableDebugStatements = $aCMDResult['enable-debug-statements'];
} else {
$this->bEnableDebugStatements = false;
}
if (isset($aCMDResult['enable-diff-updates'])) {
$this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates'];
} else {
$this->bEnableDiffUpdates = false;
}
$this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop'];
$this->oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
if ($this->bQuiet) {
$this->oNominatimCmd->addParams('--quiet');
}
if ($this->bVerbose) {
$this->oNominatimCmd->addParams('--verbose');
}
}
public function calculatePostcodes($bCMDResultAll)
{
info('Calculate Postcodes');
$this->pgsqlRunScriptFile(CONST_SqlDir.'/postcode_tables.sql');
$sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz';
if (file_exists($sPostcodeFilename)) {
$this->pgsqlRunScriptFile($sPostcodeFilename);
} else {
warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
}
$sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz';
if (file_exists($sPostcodeFilename)) {
$this->pgsqlRunScriptFile($sPostcodeFilename);
} else {
warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
}
$this->db()->exec('TRUNCATE location_postcode');
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
$sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,";
$sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))';
$sSQL .= ' FROM placex';
$sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
$sSQL .= ' AND geometry IS NOT null';
$sSQL .= ' GROUP BY country_code, pc';
$this->db()->exec($sSQL);
// only add postcodes that are not yet available in OSM
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
$sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)';
$sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'us')";
$this->db()->exec($sSQL);
// add missing postcodes for GB (if available)
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
$sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'gb')";
$this->db()->exec($sSQL);
if (!$bCMDResultAll) {
$sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
$sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
$this->db()->exec($sSQL);
}
$sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
$sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
$this->db()->exec($sSQL);
}
/**
* Return the connection to the database.
*
* @return Database object.
*
* Creates a new connection if none exists yet. Otherwise reuses the
* already established connection.
*/
private function db()
{
if (is_null($this->oDB)) {
$this->oDB = new \Nominatim\DB();
$this->oDB->connect();
}
return $this->oDB;
}
private function pgsqlRunScript($sScript, $bfatal = true)
{
runSQLScript(
$sScript,
$bfatal,
$this->bVerbose,
$this->sIgnoreErrors
);
}
public function createSqlFunctions()
{
$oCmd = (clone($this->oNominatimCmd))
->addParams('refresh', '--functions');
if (!$this->bEnableDiffUpdates) {
$oCmd->addParams('--no-diff-updates');
}
if ($this->bEnableDebugStatements) {
$oCmd->addParams('--enable-debug-statements');
}
$oCmd->run(!$this->sIgnoreErrors);
}
private function pgsqlRunScriptFile($sFilename)
{
if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
$oCmd = (new \Nominatim\Shell('psql'))
->addParams('--port', $this->aDSNInfo['port'])
->addParams('--dbname', $this->aDSNInfo['database']);
if (!$this->bVerbose) {
$oCmd->addParams('--quiet');
}
if (isset($this->aDSNInfo['hostspec'])) {
$oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
}
if (isset($this->aDSNInfo['username'])) {
$oCmd->addParams('--username', $this->aDSNInfo['username']);
}
if (isset($this->aDSNInfo['password'])) {
$oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
}
$ahGzipPipes = null;
if (preg_match('/\\.gz$/', $sFilename)) {
$aDescriptors = array(
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
);
$oZcatCmd = new \Nominatim\Shell('zcat', $sFilename);
$hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes);
if (!is_resource($hGzipProcess)) fail('unable to start zcat');
$aReadPipe = $ahGzipPipes[1];
fclose($ahGzipPipes[0]);
} else {
$oCmd->addParams('--file', $sFilename);
$aReadPipe = array('pipe', 'r');
}
$aDescriptors = array(
0 => $aReadPipe,
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
);
$ahPipes = null;
$hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
if (!is_resource($hProcess)) fail('unable to start pgsql');
// TODO: error checking
while (!feof($ahPipes[1])) {
echo fread($ahPipes[1], 4096);
}
fclose($ahPipes[1]);
$iReturn = proc_close($hProcess);
if ($iReturn > 0) {
fail("pgsql returned with error code ($iReturn)");
}
if ($ahGzipPipes) {
fclose($ahGzipPipes[1]);
proc_close($hGzipProcess);
}
}
private function replaceSqlPatterns($sSql)
{
$sSql = str_replace('{www-user}', getSetting('DATABASE_WEBUSER'), $sSql);
$aPatterns = array(
'{ts:address-data}' => getSetting('TABLESPACE_ADDRESS_DATA'),
'{ts:address-index}' => getSetting('TABLESPACE_ADDRESS_INDEX'),
'{ts:search-data}' => getSetting('TABLESPACE_SEARCH_DATA'),
'{ts:search-index}' => getSetting('TABLESPACE_SEARCH_INDEX'),
'{ts:aux-data}' => getSetting('TABLESPACE_AUX_DATA'),
'{ts:aux-index}' => getSetting('TABLESPACE_AUX_INDEX')
);
foreach ($aPatterns as $sPattern => $sTablespace) {
if ($sTablespace) {
$sSql = str_replace($sPattern, 'TABLESPACE "'.$sTablespace.'"', $sSql);
} else {
$sSql = str_replace($sPattern, '', $sSql);
}
}
return $sSql;
}
}

15
lib-php/setup_functions.php
View File

@ -1,20 +1,5 @@
<?php
function checkInFile($sOSMFile)
{
if (!isset($sOSMFile)) {
fail('missing --osm-file for data import');
}
if (!file_exists($sOSMFile)) {
fail('the path supplied to --osm-file does not exist');
}
if (!is_readable($sOSMFile)) {
fail('osm-file "' . $aCMDResult['osm-file'] . '" not readable');
}
}
function getOsm2pgsqlBinary()
{
$sBinary = getSetting('OSM2PGSQL_BINARY');

2
nominatim/cli.py
View File

@ -273,8 +273,6 @@ def get_set_parser(**kwargs):
else:
parser.parser.epilog = 'php-cgi not found. Query commands not available.'
parser.add_subcommand('transition', clicmd.AdminTransition)
return parser

1
nominatim/clicmd/__init__.py
View File

@ -9,5 +9,4 @@ from nominatim.clicmd.index import UpdateIndex
from nominatim.clicmd.refresh import UpdateRefresh
from nominatim.clicmd.admin import AdminFuncs
from nominatim.clicmd.freeze import SetupFreeze
from nominatim.clicmd.transition import AdminTransition
from nominatim.clicmd.special_phrases import ImportSpecialPhrases

146
nominatim/clicmd/transition.py
View File

@ -1,146 +0,0 @@
"""
Implementation of the 'transition' subcommand.
This subcommand provides standins for functions that were available
through the PHP scripts but are now no longer directly accessible.
This module will be removed as soon as the transition phase is over.
"""
import logging
from pathlib import Path
from nominatim.db.connection import connect
from nominatim.db import status
from nominatim.errors import UsageError
# Do not repeat documentation of subcommand classes.
# pylint: disable=C0111
# Using non-top-level imports to avoid eventually unused imports.
# pylint: disable=E0012,C0415
LOG = logging.getLogger()
class AdminTransition:
"""\
Internal functions for code transition. Do not use.
"""
@staticmethod
def add_args(parser):
group = parser.add_argument_group('Sub-functions')
group.add_argument('--create-db', action='store_true',
help='Create nominatim db')
group.add_argument('--setup-db', action='store_true',
help='Build a blank nominatim db')
group.add_argument('--import-data', action='store_true',
help='Import a osm file')
group.add_argument('--load-data', action='store_true',
help='Copy data to live tables from import table')
group.add_argument('--create-tables', action='store_true',
help='Create main tables')
group.add_argument('--create-partition-tables', action='store_true',
help='Create required partition tables')
group.add_argument('--index', action='store_true',
help='Index the data')
group.add_argument('--create-search-indices', action='store_true',
help='Create additional indices required for search and update')
group.add_argument('--create-country-names', action='store_true',
help='Create search index for default country names.')
group = parser.add_argument_group('Options')
group.add_argument('--no-partitions', action='store_true',
help='Do not partition search indices')
group.add_argument('--osm-file', metavar='FILE',
help='File to import')
group.add_argument('--drop', action='store_true',
help='Drop tables needed for updates, making the database readonly')
group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
help='Size of cache to be used by osm2pgsql (in MB)')
group.add_argument('--no-analyse', action='store_true',
help='Do not perform analyse operations during index')
group.add_argument('--ignore-errors', action='store_true',
help="Ignore certain erros on import.")
group.add_argument('--reverse-only', action='store_true',
help='Do not create search tables and indexes')
group.add_argument('--tiger-data', metavar='FILE',
help='File to import')
@staticmethod
def run(args): # pylint: disable=too-many-statements
from ..tools import database_import, tiger_data
from ..tools import refresh
if args.create_db:
LOG.warning('Create DB')
database_import.create_db(args.config.get_libpq_dsn())
if args.setup_db:
LOG.warning('Setup DB')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.setup_extensions(conn)
database_import.install_module(args.module_dir, args.project_dir,
args.config.DATABASE_MODULE_PATH,
conn=conn)
database_import.import_base_data(args.config.get_libpq_dsn(),
args.data_dir, args.no_partitions)
if args.import_data:
LOG.warning('Import data')
if not args.osm_file:
raise UsageError('Missing required --osm-file argument')
database_import.import_osm_data(Path(args.osm_file),
args.osm2pgsql_options(0, 1),
drop=args.drop,
ignore_errors=args.ignore_errors)
if args.create_tables:
LOG.warning('Create Tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_tables(conn, args.config, args.sqllib_dir, args.reverse_only)
refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
refresh.create_functions(conn, args.config, args.sqllib_dir,
enable_diff_updates=False)
database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
if args.create_partition_tables:
LOG.warning('Create Partition Tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
if args.load_data:
LOG.warning('Load data')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.truncate_data_tables(conn, args.config.MAX_WORD_FREQUENCY)
database_import.load_data(args.config.get_libpq_dsn(),
args.data_dir,
args.threads or 1)
with connect(args.config.get_libpq_dsn()) as conn:
try:
status.set_status(conn, status.compute_database_date(conn))
except Exception as exc: # pylint: disable=broad-except
LOG.error('Cannot determine date of database: %s', exc)
if args.index:
LOG.warning('Indexing')
from ..indexer.indexer import Indexer
indexer = Indexer(args.config.get_libpq_dsn(), args.threads or 1)
indexer.index_full()
if args.create_search_indices:
LOG.warning('Create Search indices')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_search_indices(conn, args.config, args.sqllib_dir, args.drop)
if args.tiger_data:
LOG.warning('Tiger data')
tiger_data.add_tiger_data(args.config.get_libpq_dsn(),
args.tiger_data,
args.threads or 1,
args.config,
args.sqllib_dir)
if args.create_country_names:
LOG.warning('Create search index for default country names.')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_country_names(conn, args.config)