mirror of
https://github.com/osm-search/Nominatim.git
synced 2024-11-23 05:35:13 +03:00
format change revert, removed bogus CL options, SetupClass to a new dir
This commit is contained in:
ThomasBarris
parent
aa6ac5a751
commit
d10f63b666
42
lib/SetupClass.php → lib/classes/SetupClass.php
Normal file → Executable file
42
lib/SetupClass.php → lib/classes/SetupClass.php
Normal file → Executable file
@ -45,10 +45,24 @@ class SetupFunctions
|
||||
|
||||
// setting member variables based on command line options stored in $aCMDResult
|
||||
$this->sVerbose = $aCMDResult['verbose'];
|
||||
$this->sIgnoreErrors = $aCMDResult['ignore-errors'];
|
||||
$this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates'];
|
||||
$this->bEnableDebugStatements = $aCMDResult['enable-debug-statements'];
|
||||
$this->bNoPartitions = $aCMDResult['no-partitions'];
|
||||
|
||||
//setting default values which are not set by the update.php array
|
||||
if (isset($aCMDResult['ignore-errors'])) {
|
||||
$this->sIgnoreErrors = $aCMDResult['ignore-errors'];
|
||||
} else {
|
||||
$this->sIgnoreErrors = false;
|
||||
}
|
||||
if (isset($aCMDResult['enable-debug-statements'])) {
|
||||
$this->bEnableDebugStatements = $aCMDResult['enable-debug-statements'];
|
||||
} else {
|
||||
$this->bEnableDebugStatements = false;
|
||||
}
|
||||
if (isset($aCMDResult['no-partitions'])) {
|
||||
$this->bNoPartitions = $aCMDResult['no-partitions'];
|
||||
} else {
|
||||
$this->bNoPartitions = false;
|
||||
}
|
||||
}
|
||||
|
||||
public function createDB()
|
||||
@ -61,11 +75,11 @@ class SetupFunctions
|
||||
|
||||
$sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database'];
|
||||
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
|
||||
$sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username'];
|
||||
$sCreateDBCmd .= ' -U '.$this->aDSNInfo['username'];
|
||||
}
|
||||
|
||||
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
|
||||
$sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec'];
|
||||
$sCreateDBCmd .= ' -h '.$this->aDSNInfo['hostspec'];
|
||||
}
|
||||
|
||||
$aProcEnv = null;
|
||||
@ -184,10 +198,10 @@ class SetupFunctions
|
||||
$osm2pgsql .= ' -C '.$this->iCacheMemory;
|
||||
$osm2pgsql .= ' -P '.$this->aDSNInfo['port'];
|
||||
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
|
||||
$osm2pgsql .= ' -U ' . $this->aDSNInfo['username'];
|
||||
$osm2pgsql .= ' -U '.$this->aDSNInfo['username'];
|
||||
}
|
||||
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
|
||||
$osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec'];
|
||||
$osm2pgsql .= ' -H '.$this->aDSNInfo['hostspec'];
|
||||
}
|
||||
$aProcEnv = null;
|
||||
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
|
||||
@ -402,7 +416,7 @@ class SetupFunctions
|
||||
// PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK,
|
||||
// PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE,
|
||||
// PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR
|
||||
echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n";
|
||||
// echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n";
|
||||
if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) {
|
||||
$resultError = pg_result_error($hPGresult);
|
||||
echo '-- error text ' . $i . ': ' . $resultError . "\n";
|
||||
@ -566,10 +580,10 @@ class SetupFunctions
|
||||
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '
|
||||
.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile;
|
||||
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
|
||||
$sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec'];
|
||||
$sBaseCmd .= ' -H '.$this->aDSNInfo['hostspec'];
|
||||
}
|
||||
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
|
||||
$sBaseCmd .= ' -U ' . $this->aDSNInfo['username'];
|
||||
$sBaseCmd .= ' -U '.$this->aDSNInfo['username'];
|
||||
}
|
||||
$aProcEnv = null;
|
||||
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
|
||||
@ -708,10 +722,10 @@ class SetupFunctions
|
||||
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
|
||||
$sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
|
||||
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
|
||||
$sCMD .= ' -h ' . $this->aDSNInfo['hostspec'];
|
||||
$sCMD .= ' -h '.$this->aDSNInfo['hostspec'];
|
||||
}
|
||||
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
|
||||
$sCMD .= ' -U ' . $this->aDSNInfo['username'];
|
||||
$sCMD .= ' -U '.$this->aDSNInfo['username'];
|
||||
}
|
||||
$aProcEnv = null;
|
||||
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
|
||||
@ -781,10 +795,10 @@ class SetupFunctions
|
||||
$sCMD .= ' -q';
|
||||
}
|
||||
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
|
||||
$sCMD .= ' -h ' . $this->aDSNInfo['hostspec'];
|
||||
$sCMD .= ' -h '.$this->aDSNInfo['hostspec'];
|
||||
}
|
||||
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
|
||||
$sCMD .= ' -U ' . $this->aDSNInfo['username'];
|
||||
$sCMD .= ' -U '.$this->aDSNInfo['username'];
|
||||
}
|
||||
$aProcEnv = null;
|
||||
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
|
||||
@ -118,9 +118,6 @@ function createUpdateArgvArray()
|
||||
|
||||
array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'),
|
||||
array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'),
|
||||
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
|
||||
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
|
||||
array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
|
||||
);
|
||||
return $aCMDOptions;
|
||||
}
|
||||
|
||||
@ -1,13 +1,12 @@
|
||||
#!@PHP_BIN@ -Cq
|
||||
<?php
|
||||
|
||||
require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
|
||||
require_once CONST_BasePath . '/lib/init-cmd.php';
|
||||
require_once CONST_BasePath . '/lib/SetupClass.php';
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
require_once(CONST_BasePath.'/lib/classes/SetupClass.php');
|
||||
// ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt
|
||||
// ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB()
|
||||
|
||||
require_once CONST_BasePath . '/lib/setup_functions.php';
|
||||
require_once(CONST_BasePath.'/lib/setup_functions.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
use Nominatim\Setup\SetupFunctions as SetupFunctions;
|
||||
@ -25,7 +24,6 @@ $bDidSomething = false;
|
||||
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
|
||||
// to remain in /lib/setup_functions.php function
|
||||
checkInFile($aCMDResult['osm-file']);
|
||||
echo $aCMDResult['osm-file'];
|
||||
}
|
||||
|
||||
// osmosis init is no longer supported
|
||||
|
||||
157
utils/update.php
157
utils/update.php
@ -1,42 +1,35 @@
|
||||
#!@PHP_BIN@ -Cq
|
||||
<?php
|
||||
|
||||
require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
|
||||
require_once CONST_BasePath . '/lib/init-cmd.php';
|
||||
require_once CONST_BasePath . '/lib/setup_functions.php';
|
||||
require_once CONST_BasePath . '/lib/SetupClass.php';
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
require_once(CONST_BasePath.'/lib/setup_functions.php');
|
||||
require_once(CONST_BasePath.'/lib/classes/SetupClass.php');
|
||||
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
use Nominatim\Setup\SetupFunctions as SetupFunctions;
|
||||
|
||||
$aCMDOptions = createUpdateArgvArray();
|
||||
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||
|
||||
if (!isset($aResult['index-instances'])) {
|
||||
$aResult['index-instances'] = 1;
|
||||
}
|
||||
|
||||
if (!isset($aResult['index-rank'])) {
|
||||
$aResult['index-rank'] = 0;
|
||||
}
|
||||
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
|
||||
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
|
||||
|
||||
date_default_timezone_set('Etc/UTC');
|
||||
|
||||
$oDB = &getDB();
|
||||
$oDB =& getDB();
|
||||
|
||||
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
|
||||
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) {
|
||||
$aDSNInfo['port'] = 5432;
|
||||
}
|
||||
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
|
||||
|
||||
// cache memory to be used by osm2pgsql, should not be more than the available memory
|
||||
$iCacheMemory = (isset($aResult['osm2pgsql-cache']) ? $aResult['osm2pgsql-cache'] : 2000);
|
||||
$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
|
||||
if ($iCacheMemory + 500 > getTotalMemoryMB()) {
|
||||
$iCacheMemory = getCacheMemoryMB();
|
||||
echo "WARNING: resetting cache memory to $iCacheMemory\n";
|
||||
}
|
||||
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'];
|
||||
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
|
||||
if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
|
||||
$sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username'];
|
||||
}
|
||||
@ -49,12 +42,12 @@ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
|
||||
}
|
||||
|
||||
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
|
||||
$sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File;
|
||||
$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
|
||||
}
|
||||
|
||||
if ($aResult['init-updates']) {
|
||||
// sanity check that the replication URL is correct
|
||||
$sBaseState = file_get_contents(CONST_Replication_Url . '/state.txt');
|
||||
$sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt');
|
||||
if ($sBaseState === false) {
|
||||
echo "\nCannot find state.txt file at the configured replication URL.\n";
|
||||
echo "Does the URL point to a directory containing OSM update data?\n\n";
|
||||
@ -68,7 +61,7 @@ if ($aResult['init-updates']) {
|
||||
fail('CONST_Pyosmium_Binary not configured');
|
||||
}
|
||||
$aOutput = 0;
|
||||
$sCmd = CONST_Pyosmium_Binary . ' --help';
|
||||
$sCmd = CONST_Pyosmium_Binary.' --help';
|
||||
exec($sCmd, $aOutput, $iRet);
|
||||
if ($iRet != 0) {
|
||||
echo "Cannot execute pyosmium-get-changes.\n";
|
||||
@ -78,8 +71,11 @@ if ($aResult['init-updates']) {
|
||||
}
|
||||
|
||||
if (!$aResult['no-update-functions']) {
|
||||
// instatiate setupClass to use the function therein
|
||||
// instantiate Setup class
|
||||
// Try accessing the C module,
|
||||
if (!checkModulePresence()) {
|
||||
fail('error loading nominatim.so module');
|
||||
}
|
||||
// instantiate setupClass to use the function therein
|
||||
$cSetup = new SetupFunctions($aResult);
|
||||
$cSetup->createFunctions();
|
||||
}
|
||||
@ -88,11 +84,11 @@ if ($aResult['init-updates']) {
|
||||
if ($sDatabaseDate === false) {
|
||||
fail('Cannot determine date of database.');
|
||||
}
|
||||
$sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3 * 60 * 60));
|
||||
$sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));
|
||||
|
||||
// get the appropriate state id
|
||||
$aOutput = 0;
|
||||
$sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url;
|
||||
$sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url;
|
||||
exec($sCmd, $aOutput, $iRet);
|
||||
if ($iRet != 0 || $aOutput[0] == 'None') {
|
||||
fail('Error running pyosmium tools');
|
||||
@ -100,7 +96,7 @@ if ($aResult['init-updates']) {
|
||||
|
||||
pg_query($oDB->connection, 'TRUNCATE import_status');
|
||||
$sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
|
||||
$sSQL .= $sDatabaseDate . "'," . $aOutput[0] . ', true)';
|
||||
$sSQL .= $sDatabaseDate."',".$aOutput[0].', true)';
|
||||
if (!pg_query($oDB->connection, $sSQL)) {
|
||||
fail('Could not enter sequence into database.');
|
||||
}
|
||||
@ -115,7 +111,7 @@ if ($aResult['check-for-updates']) {
|
||||
fail('Updates not set up. Please run ./utils/update.php --init-updates.');
|
||||
}
|
||||
|
||||
system(CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState['sequence_id'], $iRet);
|
||||
system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet);
|
||||
exit($iRet);
|
||||
}
|
||||
|
||||
@ -128,8 +124,8 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
|
||||
}
|
||||
|
||||
// Import the file
|
||||
$sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile;
|
||||
echo $sCMD . "\n";
|
||||
$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
|
||||
echo $sCMD."\n";
|
||||
$iErrorLevel = runWithEnv($sCMD, $aProcEnv);
|
||||
|
||||
if ($iErrorLevel) {
|
||||
@ -141,35 +137,35 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
|
||||
|
||||
if ($aResult['calculate-postcodes']) {
|
||||
info('Update postcodes centroids');
|
||||
$sTemplate = file_get_contents(CONST_BasePath . '/sql/update-postcodes.sql');
|
||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
|
||||
runSQLScript($sTemplate, true, true);
|
||||
}
|
||||
|
||||
$sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc';
|
||||
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
|
||||
$bHaveDiff = false;
|
||||
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
|
||||
$sContentURL = '';
|
||||
if (isset($aResult['import-node']) && $aResult['import-node']) {
|
||||
if ($bUseOSMApi) {
|
||||
$sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult['import-node'];
|
||||
$sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
|
||||
} else {
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult['import-node'] . ');out%20meta;';
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($aResult['import-way']) && $aResult['import-way']) {
|
||||
if ($bUseOSMApi) {
|
||||
$sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult['import-way'] . '/full';
|
||||
$sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
|
||||
} else {
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult['import-way'] . ');node(w););out%20meta;';
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($aResult['import-relation']) && $aResult['import-relation']) {
|
||||
if ($bUseOSMApi) {
|
||||
$sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult['import-relation'] . '/full';
|
||||
$sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
|
||||
} else {
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult['import-relation'] . ');way(r);node(w));node(r));out%20meta;';
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
|
||||
}
|
||||
}
|
||||
|
||||
@ -180,8 +176,8 @@ if ($sContentURL) {
|
||||
|
||||
if ($bHaveDiff) {
|
||||
// import generated change file
|
||||
$sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile;
|
||||
echo $sCMD . "\n";
|
||||
$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
|
||||
echo $sCMD."\n";
|
||||
$iErrorLevel = runWithEnv($sCMD, $aProcEnv);
|
||||
if ($iErrorLevel) {
|
||||
fail("osm2pgsql exited with error level $iErrorLevel\n");
|
||||
@ -189,7 +185,7 @@ if ($bHaveDiff) {
|
||||
}
|
||||
|
||||
if ($aResult['deduplicate']) {
|
||||
$oDB = &getDB();
|
||||
$oDB =& getDB();
|
||||
|
||||
if (getPostgresVersion($oDB) < 9.3) {
|
||||
fail('ERROR: deduplicate is only currently supported in postgresql 9.3');
|
||||
@ -201,7 +197,7 @@ if ($aResult['deduplicate']) {
|
||||
|
||||
// we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
|
||||
foreach ($aPartitions as $i => $sPartition) {
|
||||
$sSQL = 'select count(*) from search_name_' . $sPartition;
|
||||
$sSQL = 'select count(*) from search_name_'.$sPartition;
|
||||
$nEntries = chksql($oDB->getOne($sSQL));
|
||||
if ($nEntries == 0) {
|
||||
unset($aPartitions[$i]);
|
||||
@ -213,14 +209,11 @@ if ($aResult['deduplicate']) {
|
||||
$sSQL .= ' group by word_token having count(*) > 1 order by word_token';
|
||||
$aDuplicateTokens = chksql($oDB->getAll($sSQL));
|
||||
foreach ($aDuplicateTokens as $aToken) {
|
||||
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') {
|
||||
continue;
|
||||
}
|
||||
|
||||
echo 'Deduping ' . $aToken['word_token'] . "\n";
|
||||
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
|
||||
echo 'Deduping '.$aToken['word_token']."\n";
|
||||
$sSQL = 'select word_id,';
|
||||
$sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';
|
||||
$sSQL .= " from word where word_token = '" . $aToken['word_token'];
|
||||
$sSQL .= " from word where word_token = '".$aToken['word_token'];
|
||||
$sSQL .= "' and class is null and type is null and country_code is null order by num desc";
|
||||
$aTokenSet = chksql($oDB->getAll($sSQL));
|
||||
|
||||
@ -229,34 +222,34 @@ if ($aResult['deduplicate']) {
|
||||
|
||||
foreach ($aTokenSet as $aRemove) {
|
||||
$sSQL = 'update search_name set';
|
||||
$sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . '),';
|
||||
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
$sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),';
|
||||
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = 'update search_name set';
|
||||
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']';
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = 'update location_area_country set';
|
||||
$sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
$sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
foreach ($aPartitions as $sPartition) {
|
||||
$sSQL = 'update search_name_' . $sPartition . ' set';
|
||||
$sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
$sSQL = 'update search_name_'.$sPartition.' set';
|
||||
$sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = 'update location_area_country set';
|
||||
$sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
$sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
|
||||
chksql($oDB->query($sSQL));
|
||||
}
|
||||
|
||||
$sSQL = 'delete from word where word_id = ' . $aRemove['word_id'];
|
||||
$sSQL = 'delete from word where word_id = '.$aRemove['word_id'];
|
||||
chksql($oDB->query($sSQL));
|
||||
}
|
||||
}
|
||||
@ -264,12 +257,12 @@ if ($aResult['deduplicate']) {
|
||||
|
||||
if ($aResult['recompute-word-counts']) {
|
||||
info('Recompute frequency of full-word search terms');
|
||||
$sTemplate = file_get_contents(CONST_BasePath . '/sql/words_from_search_name.sql');
|
||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql');
|
||||
runSQLScript($sTemplate, true, true);
|
||||
}
|
||||
|
||||
if ($aResult['index']) {
|
||||
$sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'] . ' -r ' . $aResult['index-rank'];
|
||||
$sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank'];
|
||||
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
|
||||
$sCmd .= ' -H ' . $aDSNInfo['hostspec'];
|
||||
}
|
||||
@ -284,13 +277,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
//
|
||||
if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
|
||||
fail('Error: Update interval too low for download.geofabrik.de. ' .
|
||||
"Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
|
||||
"Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
|
||||
}
|
||||
|
||||
$sImportFile = CONST_InstallPath . '/osmosischange.osc';
|
||||
$sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size;
|
||||
$sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile;
|
||||
$sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'];
|
||||
$sImportFile = CONST_InstallPath.'/osmosischange.osc';
|
||||
$sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
|
||||
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
|
||||
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
|
||||
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
|
||||
$sCMDIndex .= ' -H ' . $aDSNInfo['hostspec'];
|
||||
}
|
||||
@ -307,7 +300,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
echo 'Currently at sequence ' . $aLastState['sequence_id'] . ' (' . $aLastState['lastimportdate'] . ') - ' . $aLastState['indexed'] . " indexed\n";
|
||||
echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n";
|
||||
|
||||
$sBatchEnd = $aLastState['lastimportdate'];
|
||||
$iEndSequence = $aLastState['sequence_id'];
|
||||
@ -330,23 +323,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
if (file_exists($sImportFile)) {
|
||||
unlink($sImportFile);
|
||||
}
|
||||
exec($sCMDDownload . ' -I ' . $iNextSeq, $aOutput, $iResult);
|
||||
exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
|
||||
|
||||
if ($iResult == 3) {
|
||||
echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec.\n";
|
||||
echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
|
||||
sleep(CONST_Replication_Recheck_Interval);
|
||||
} elseif ($iResult != 0) {
|
||||
echo 'ERROR: updates failed.';
|
||||
exit($iResult);
|
||||
} else {
|
||||
$iEndSequence = (int) $aOutput[0];
|
||||
$iEndSequence = (int)$aOutput[0];
|
||||
}
|
||||
} while ($iResult);
|
||||
|
||||
// get the newest object from the diff file
|
||||
$sBatchEnd = 0;
|
||||
$iRet = 0;
|
||||
exec(CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile, $sBatchEnd, $iRet);
|
||||
exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
|
||||
if ($iRet == 5) {
|
||||
echo "Diff file is empty. skipping import.\n";
|
||||
if (!$aResult['import-osmosis-all']) {
|
||||
@ -362,7 +355,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
|
||||
// Import the file
|
||||
$fCMDStartTime = time();
|
||||
echo $sCMDImport . "\n";
|
||||
echo $sCMDImport."\n";
|
||||
unset($sJunk);
|
||||
$iErrorLevel = runWithEnv($sCMDImport, $aProcEnv);
|
||||
if ($iErrorLevel) {
|
||||
@ -375,8 +368,8 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
$sSQL = 'INSERT INTO import_osmosis_log';
|
||||
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
|
||||
$sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
|
||||
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','";
|
||||
$sSQL .= date('Y-m-d H:i:s') . "','import')";
|
||||
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
|
||||
$sSQL .= date('Y-m-d H:i:s')."','import')";
|
||||
var_Dump($sSQL);
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
@ -384,7 +377,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
$sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence";
|
||||
var_Dump($sSQL);
|
||||
chksql($oDB->query($sSQL));
|
||||
echo date('Y-m-d H:i:s') . " Completed download step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n";
|
||||
echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
|
||||
}
|
||||
|
||||
// Index file
|
||||
@ -402,20 +395,18 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
$sSQL = 'INSERT INTO import_osmosis_log';
|
||||
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
|
||||
$sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
|
||||
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','";
|
||||
$sSQL .= date('Y-m-d H:i:s') . "','index')";
|
||||
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
|
||||
$sSQL .= date('Y-m-d H:i:s')."','index')";
|
||||
var_Dump($sSQL);
|
||||
$oDB->query($sSQL);
|
||||
echo date('Y-m-d H:i:s') . " Completed index step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n";
|
||||
echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
|
||||
|
||||
$sSQL = 'update import_status set indexed = true';
|
||||
$oDB->query($sSQL);
|
||||
}
|
||||
|
||||
$fDuration = time() - $fStartTime;
|
||||
echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n";
|
||||
if (!$aResult['import-osmosis-all']) {
|
||||
exit(0);
|
||||
}
|
||||
echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n";
|
||||
if (!$aResult['import-osmosis-all']) exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user