osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-11-28 05:48:36 +03:00
Code Issues Packages Projects Releases Wiki Activity
1461dc603e
Nominatim/utils/update.php

522 lines
18 KiB
PHP
Raw Normal View History

diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
#!/usr/bin/php -Cq
<?php
retry loop on osmosis
2011-04-27 16:14:33 +04:00
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
ini_set('memory_limit', '800M');
$aCMDOptions = array(
"Import / update / index osm data",
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('max-load', '', 0, 1, 1, 1, 'float', 'Maximum load average - indexing is paused if this is exceeded'),
array('max-blocking', '', 0, 1, 1, 1, 'int', 'Maximum blocking processes - indexing is aborted / paused if this is exceeded'),
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
array('no-npi', '', 0, 1, 0, 0, 'bool', 'Do not write npi index files'),
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
array('import-npi-all', '', 0, 1, 0, 0, 'bool', 'Import npi pre-indexed files'),
array('import-hourly', '', 0, 1, 0, 0, 'bool', 'Import hourly diffs'),
array('import-daily', '', 0, 1, 0, 0, 'bool', 'Import daily diffs'),
array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
array('index-estrate', '', 0, 1, 1, 1, 'int', 'Estimated indexed items per second (def:30)'),
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
);
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
if ($aResult['import-hourly'] + $aResult['import-daily'] + isset($aResult['import-diff']) > 1)
{
showUsage($aCMDOptions, true, 'Select either import of hourly or daily');
}
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
/*
// Lock to prevent multiple copies running
if (exec('/bin/ps uww | grep '.basename(__FILE__).' | grep -v /dev/null | grep -v grep -c', $aOutput2, $iResult) > 1)
{
echo "Copy already running\n";
exit;
}
if (!isset($aResult['max-load'])) $aResult['max-load'] = 1.9;
if (!isset($aResult['max-blocking'])) $aResult['max-blocking'] = 3;
if (getBlockingProcesses() > $aResult['max-blocking'])
{
echo "Too many blocking processes for import\n";
exit;
}
*/
// Assume osm2pgsql is in the folder above
$sBasePath = dirname(dirname(__FILE__));
$oDB =& getDB();
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
$bFirst = true;
$bContinue = $aResult['import-all'];
while ($bContinue || $bFirst)
{
$bFirst = false;
if ($aResult['import-hourly'])
{
// Mirror the hourly diffs
exec('wget --quiet --mirror -l 1 -P '.$sMirrorDir.' http://planet.openstreetmap.org/hourly');
$sNextFile = $oDB->getOne('select TO_CHAR(lastimportdate,\'YYYYMMDDHH24\')||\'-\'||TO_CHAR(lastimportdate+\'1 hour\'::interval,\'YYYYMMDDHH24\')||\'.osc.gz\' from import_status');
$sNextFile = $sMirrorDir.'planet.openstreetmap.org/hourly/'.$sNextFile;
$sUpdateSQL = 'update import_status set lastimportdate = lastimportdate+\'1 hour\'::interval';
}
if ($aResult['import-daily'])
{
// Mirror the daily diffs
exec('wget --quiet --mirror -l 1 -P '.$sMirrorDir.' http://planet.openstreetmap.org/daily');
$sNextFile = $oDB->getOne('select TO_CHAR(lastimportdate,\'YYYYMMDD\')||\'-\'||TO_CHAR(lastimportdate+\'1 day\'::interval,\'YYYYMMDD\')||\'.osc.gz\' from import_status');
$sNextFile = $sMirrorDir.'planet.openstreetmap.org/daily/'.$sNextFile;
$sUpdateSQL = 'update import_status set lastimportdate = lastimportdate::date + 1';
}
if (isset($aResult['import-diff']))
{
// import diff directly (e.g. from osmosis --rri)
$sNextFile = $aResult['import-diff'];
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
if (!file_exists($sNextFile))
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
{
echo "Cannot open $nextFile\n";
exit;
}
// Don't update the import status - we don't know what this file contains
$sUpdateSQL = 'update import_status set lastimportdate = now() where false';
}
// Missing file is not an error - it might not be created yet
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
if (($aResult['import-hourly'] || $aResult['import-daily'] || isset($aResult['import-diff'])) && file_exists($sNextFile))
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
{
// Import the file
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
$sCMD = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sNextFile;
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
echo $sCMD."\n";
exec($sCMD, $sJunk, $iErrorLevel);
if ($iErrorLevel)
{
echo "Error from osm2pgsql, $iErrorLevel\n";
exit;
}
// Move the date onwards
$oDB->query($sUpdateSQL);
}
else
{
$bContinue = false;
}
}
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
$bModifyXML = false;
$sModifyXMLstr = '';
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
if (isset($aResult['import-file']) && $aResult['import-file'])
{
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
$bModifyXML = true;
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
}
if (isset($aResult['import-node']) && $aResult['import-node'])
{
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
$bModifyXML = true;
$sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']);
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
}
if (isset($aResult['import-way']) && $aResult['import-way'])
{
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
$bModifyXML = true;
$sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full');
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
}
if (isset($aResult['import-relation']) && $aResult['import-relation'])
{
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
$bModifyXML = true;
$sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full');
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
}
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
if ($bModifyXML)
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
{
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
// derive change from normal osm file with osmosis
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
if ($aResult['import-file'])
{
$sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
echo $sCMD."\n";
exec($sCMD, $sJunk, $iErrorLevel);
if ($iErrorLevel)
{
echo "Error converting osm to osc, osmosis returned: $iErrorLevel\n";
exit;
}
}
else
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
{
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
$aSpec = array(
0 => array("pipe", "r"), // stdin
1 => array("pipe", "w"), // stdout
2 => array("pipe", "w") // stderr
);
$sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
echo $sCMD."\n";
$hProc = proc_open($sCMD, $aSpec, $aPipes);
if (!is_resource($hProc))
{
echo "Error converting osm to osc, osmosis failed\n";
exit;
}
fwrite($aPipes[0], $sModifyXMLstr);
fclose($aPipes[0]);
$sOut = stream_get_contents($aPipes[1]);
if ($aResult['verbose']) echo $sOut;
fclose($aPipes[1]);
$sErrors = stream_get_contents($aPipes[2]);
if ($aResult['verbose']) echo $sErrors;
fclose($aPipes[2]);
if ($iError = proc_close($hProc))
{
echo "Error converting osm to osc, osmosis returned: $iError\n";
echo $sOut;
echo $sErrors;
exit;
}
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
}
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
// import generated change file
$aPipes = array();
$sCMD = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sTemporaryFile;
echo $sCMD."\n";
exec($sCMD, $sJunk, $iErrorLevel);
if ($iErrorLevel)
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
{
make .osc files with osmosis
2012-04-20 20:29:14 +04:00
echo "osm2pgsql exited with error level $iErrorLevel\n";
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
exit;
}
}
if ($aResult['deduplicate'])
{
$oDB =& getDB();
$sSQL = 'select partition from country_name order by country_code';
$aPartitions = $oDB->getCol($sSQL);
if (PEAR::isError($aPartitions))
{
fail($aPartitions->getMessage());
}
$aPartitions[] = 0;
$sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
$aDuplicateTokens = $oDB->getAll($sSQL);
foreach($aDuplicateTokens as $aToken)
{
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
echo "Deduping ".$aToken['word_token']."\n";
$sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
$aTokenSet = $oDB->getAll($sSQL);
if (PEAR::isError($aTokenSet))
{
var_dump($aTokenSet, $sSQL);
exit;
}
$aKeep = array_shift($aTokenSet);
$iKeepID = $aKeep['word_id'];
foreach($aTokenSet as $aRemove)
{
$sSQL = "update search_name set";
$sSQL .= " name_vector = (name_vector - ".$aRemove['word_id'].")+".$iKeepID.",";
$sSQL .= " nameaddress_vector = (nameaddress_vector - ".$aRemove['word_id'].")+".$iKeepID;
$sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
$x = $oDB->query($sSQL);
if (PEAR::isError($x))
{
var_dump($x);
exit;
}
$sSQL = "update search_name set";
$sSQL .= " nameaddress_vector = (nameaddress_vector - ".$aRemove['word_id'].")+".$iKeepID;
$sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
$x = $oDB->query($sSQL);
if (PEAR::isError($x))
{
var_dump($x);
exit;
}
$sSQL = "update location_area_country set";
$sSQL .= " keywords = (keywords - ".$aRemove['word_id'].")+".$iKeepID;
$sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
$x = $oDB->query($sSQL);
if (PEAR::isError($x))
{
var_dump($x);
exit;
}
foreach ($aPartitions as $sPartition)
{
$sSQL = "update search_name_".$sPartition." set";
$sSQL .= " name_vector = (name_vector - ".$aRemove['word_id'].")+".$iKeepID.",";
$sSQL .= " nameaddress_vector = (nameaddress_vector - ".$aRemove['word_id'].")+".$iKeepID;
$sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
$x = $oDB->query($sSQL);
if (PEAR::isError($x))
{
var_dump($x);
exit;
}
$sSQL = "update search_name_".$sPartition." set";
$sSQL .= " nameaddress_vector = (nameaddress_vector - ".$aRemove['word_id'].")+".$iKeepID;
$sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
$x = $oDB->query($sSQL);
if (PEAR::isError($x))
{
var_dump($x);
exit;
}
$sSQL = "update location_area_country set";
$sSQL .= " keywords = (keywords - ".$aRemove['word_id'].")+".$iKeepID;
$sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
$x = $oDB->query($sSQL);
if (PEAR::isError($x))
{
var_dump($x);
exit;
}
}
$sSQL = "delete from word where word_id = ".$aRemove['word_id'];
$x = $oDB->query($sSQL);
if (PEAR::isError($x))
{
var_dump($x);
exit;
}
}
}
}
if ($aResult['index'])
{
index($aResult, $sDatabaseDSN);
}
if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
{
$sImportFile = CONST_BasePath.'/data/osmosischange.osc';
- remove dependency on intarray extension - new config options for postgresql version and location of osm2pgsql and osmosis binaries - new option --no-npi for update.php that supresses writing of NPIs in osmosis update mode - add more GRANTs for www-data
2012-01-21 14:21:42 +04:00
$sOsmosisCMD = CONST_Osmosis_Binary;
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
$sOsmosisConfigDirectory = CONST_BasePath.'/settings';
$sDatabaseName = 'nominatim';
$sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
- remove dependency on intarray extension - new config options for postgresql version and location of osm2pgsql and osmosis binaries - new option --no-npi for update.php that supresses writing of NPIs in osmosis update mode - add more GRANTs for www-data
2012-01-21 14:21:42 +04:00
$sCMDImport = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$sDatabaseName.' '.$sImportFile;
$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -t '.$aResult['index-instances'];
if (!$aResult['no-npi']) {
$sCMDIndex .= '-F ';
}
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
while(true)
{
$fStartTime = time();
$iFileSize = 1001;
// Logic behind this is that osm2pgsql locks the database quite a bit
// So it is better to import lots of small files
// But indexing works most efficiently on large amounts of data
// So do lots of small imports and a BIG index
// while($aResult['import-osmosis-all'] && $iFileSize > 1000)
// {
if (!file_exists($sImportFile))
{
// Use osmosis to download the file
$fCMDStartTime = time();
echo $sCMDDownload."\n";
exec($sCMDDownload, $sJunk, $iErrorLevel);
retry loop on osmosis
2011-04-27 16:14:33 +04:00
while ($iErrorLevel == 1)
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
{
echo "Error: $iErrorLevel\n";
retry loop on osmosis
2011-04-27 16:14:33 +04:00
sleep(60);
echo 'Re-trying: '.$sCMDDownload."\n";
exec($sCMDDownload, $sJunk, $iErrorLevel);
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
}
$iFileSize = filesize($sImportFile);
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
$oDB->query($sSQL);
}
$iFileSize = filesize($sImportFile);
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
// Import the file
$fCMDStartTime = time();
echo $sCMDImport."\n";
exec($sCMDImport, $sJunk, $iErrorLevel);
if ($iErrorLevel)
{
echo "Error: $iErrorLevel\n";
exit;
}
echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
var_Dump($sSQL);
$oDB->query($sSQL);
// Archive for debug?
unlink($sImportFile);
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
// }
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
// Index file
$sThisIndexCmd = $sCMDIndex;
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
if (!$aResult['no-npi'])
{
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
$fCMDStartTime = time();
$iFileID = $oDB->getOne('select nextval(\'file\')');
if (PEAR::isError($iFileID))
{
bugfix nominatim
2011-09-09 20:00:34 +04:00
echo $iFileID->getMessage()."\n";
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
exit;
}
$sFileDir = CONST_BasePath.'/export/diff/';
$sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
$sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT);
if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true);
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
$sThisIndexCmd .= $sFileDir;
$sThisIndexCmd .= '/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT);
$sThisIndexCmd .= ".npi.out";
preg_match('#^([0-9]{4})-([0-9]{2})-([0-9]{2})#', $sBatchEnd, $aBatchMatch);
$sFileDir = CONST_BasePath.'/export/index/';
$sFileDir .= $aBatchMatch[1].'/'.$aBatchMatch[2];
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true);
file_put_contents($sFileDir.'/'.$aBatchMatch[3].'.idx', "$sBatchEnd\t$iFileID\n", FILE_APPEND);
}
if (!$aResult['no-index'])
{
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
echo "$sThisIndexCmd\n";
exec($sThisIndexCmd, $sJunk, $iErrorLevel);
if ($iErrorLevel)
{
echo "Error: $iErrorLevel\n";
exit;
}
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
if (!$aResult['no-npi'])
{
- remove dependency on intarray extension - new config options for postgresql version and location of osm2pgsql and osmosis binaries - new option --no-npi for update.php that supresses writing of NPIs in osmosis update mode - add more GRANTs for www-data
2012-01-21 14:21:42 +04:00
$sFileDir = CONST_BasePath.'/export/diff/';
$sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
$sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT);
$sThisIndexCmd = 'bzip2 -z9 '.$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out";
echo "$sThisIndexCmd\n";
exec($sThisIndexCmd, $sJunk, $iErrorLevel);
if ($iErrorLevel)
{
echo "Error: $iErrorLevel\n";
exit;
}
rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2",
$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.bz2");
}
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
}
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
$oDB->query($sSQL);
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
$sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
$oDB->query($sSQL);
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
$fDuration = time() - $fStartTime;
echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n";
if (!$aResult['import-osmosis-all']) exit;
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
echo "Sleeping ".max(0,60-$fDuration)." seconds\n";
sleep(max(0,60-$fDuration));
}
DB Scheme changes: alter table placex add column calculated_country_code varchar(2); function changes: ----------------- Move to ST_PointOnSurface from ST_Centroid in various places to avoid looking up a point outside the polygon Move to ST_Covers from ST_Contains to include points on admin boundaries Re-order preference for get_country_code now our data is better. country_osm_grid is now the preffered source. Fix code to calculate country code in placex_insert, rank_search test was too early Add extra field to placex 'calculated_country_code' to improve structure of code Move split_geometery function out of add_location into its own function Rewrite split_geometery to be more efficient. Change place_insert to do more updates and less delete/inserts (delete is slow) Include wikipedia links in details.php ouput Cleanup no longer used geometry validation (adding overhead) Include debug statements in function.sql (--DEBUG: ) and add flag to setup.php to turn them on setup.php: ---------- add flag --disable-token-precalc to speed up debuging add flag --index-noanalyse to disable analysising DB at rank 4 and 26 (previously removed, but on my local DB it seems to be required) add flag --enable-diff-updates (modifier to --create-functions) to turn on the code required for diff updates without having to modify functions.sql add flag --enable-debug-statements (modifier to --create-functions) to turn on debug warning statements update.php: ----------- added flag --no-index to import osmosis changes without indexing them extend the hack to allow import of JOSM generated osm files country_grid.sql - reference copy of the sql used to generate the country_osm_grid table, needs cleanup
2012-05-22 18:27:42 +04:00
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
}
if ($aResult['import-npi-all'])
{
$iNPIID = $oDB->getOne('select max(npiid) from import_npi_log');
if (PEAR::isError($iNPIID))
{
var_dump($iNPIID);
exit;
}
$sConfigDirectory = CONST_BasePath.'/settings';
$sCMDImportTemplate = $sBasePath.'/nominatim/nominatim -d gazetteer -P 5433 -I -T '.$sBasePath.'/nominatim/partitionedtags.def -F ';
while(true)
{
$fStartTime = time();
$iNPIID++;
$sImportFile = CONST_BasePath.'/export/diff/';
$sImportFile .= str_pad(floor($iNPIID/1000000), 3, '0', STR_PAD_LEFT);
$sImportFile .= '/'.str_pad(floor($iNPIID/1000) % 1000, 3, '0', STR_PAD_LEFT);
$sImportFile .= '/'.str_pad($iNPIID % 1000, 3, '0', STR_PAD_LEFT);
$sImportFile .= ".npi";
while(!file_exists($sImportFile) && !file_exists($sImportFile.'.bz2'))
{
retry loop on osmosis
2011-04-27 16:14:33 +04:00
echo "sleep (waiting for $sImportFile)\n";
diff updates and npi diff updates
2011-03-18 15:36:17 +03:00
sleep(10);
}
if (file_exists($sImportFile.'.bz2')) $sImportFile .= '.bz2';
$iFileSize = filesize($sImportFile);
// Import the file
$fCMDStartTime = time();
$sCMDImport = $sCMDImportTemplate . $sImportFile;
echo $sCMDImport."\n";
exec($sCMDImport, $sJunk, $iErrorLevel);
if ($iErrorLevel)
{
echo "Error: $iErrorLevel\n";
exit;
}
$sBatchEnd = $iNPIID;
echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
$sSQL = "INSERT INTO import_npi_log values ($iNPIID, null, $iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
var_Dump($sSQL);
$oDB->query($sSQL);
}
}
function getosmosistimestamp($sOsmosisConfigDirectory)
{
$sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
return str_replace('\:',':',$aResult[1]);
}
Reference in New Issue Copy Permalink