2011-03-18 15:36:17 +03:00
< ? php
2011-04-27 16:14:33 +04:00
2018-09-05 23:01:03 +03:00
require_once ( CONST_BasePath . '/lib/init-cmd.php' );
require_once ( CONST_BasePath . '/lib/setup_functions.php' );
2018-09-17 11:28:00 +03:00
require_once ( CONST_BasePath . '/lib/setup/SetupClass.php' );
2018-11-24 12:20:24 +03:00
require_once ( CONST_BasePath . '/lib/setup/AddressLevelParser.php' );
2018-09-05 23:01:03 +03:00
2016-09-04 04:19:48 +03:00
ini_set ( 'memory_limit' , '800M' );
2018-08-31 22:31:38 +03:00
use Nominatim\Setup\SetupFunctions as SetupFunctions ;
2018-08-24 17:15:39 +03:00
2018-09-17 11:28:00 +03:00
// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
$aCMDOptions
= array (
2018-10-02 23:59:10 +03:00
'Import / update / index osm data' ,
array ( 'help' , 'h' , 0 , 1 , 0 , 0 , false , 'Show Help' ),
array ( 'quiet' , 'q' , 0 , 1 , 0 , 0 , 'bool' , 'Quiet output' ),
array ( 'verbose' , 'v' , 0 , 1 , 0 , 0 , 'bool' , 'Verbose output' ),
array ( 'init-updates' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Set up database for updating' ),
array ( 'check-for-updates' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Check if new updates are available' ),
array ( 'no-update-functions' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)' ),
array ( 'import-osmosis' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Import updates once' ),
array ( 'import-osmosis-all' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Import updates forever' ),
array ( 'no-index' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Do not index the new data' ),
array ( 'calculate-postcodes' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Update postcode centroid table' ),
array ( 'import-file' , '' , 0 , 1 , 1 , 1 , 'realpath' , 'Re-import data from an OSM file' ),
array ( 'import-diff' , '' , 0 , 1 , 1 , 1 , 'realpath' , 'Import a diff (osc) file from local file system' ),
array ( 'osm2pgsql-cache' , '' , 0 , 1 , 1 , 1 , 'int' , 'Cache size used by osm2pgsql' ),
array ( 'import-node' , '' , 0 , 1 , 1 , 1 , 'int' , 'Re-import node' ),
array ( 'import-way' , '' , 0 , 1 , 1 , 1 , 'int' , 'Re-import way' ),
array ( 'import-relation' , '' , 0 , 1 , 1 , 1 , 'int' , 'Re-import relation' ),
array ( 'import-from-main-api' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Use OSM API instead of Overpass to download objects' ),
array ( 'index' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Index' ),
array ( 'index-rank' , '' , 0 , 1 , 1 , 1 , 'int' , 'Rank to start indexing from' ),
array ( 'index-instances' , '' , 0 , 1 , 1 , 1 , 'int' , 'Number of indexing instances (threads)' ),
array ( 'deduplicate' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Deduplicate tokens' ),
array ( 'recompute-word-counts' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Compute frequency of full-word search terms' ),
2018-11-24 12:20:24 +03:00
array ( 'update-address-levels' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Reimport address level configuration (EXPERT)' ),
2019-10-28 23:17:42 +03:00
array ( 'recompute-importance' , '' , 0 , 1 , 0 , 0 , 'bool' , 'Recompute place importances' ),
2018-10-02 23:59:10 +03:00
array ( 'no-npi' , '' , 0 , 1 , 0 , 0 , 'bool' , '(obsolete)' ),
);
2018-09-17 11:28:00 +03:00
2016-09-04 04:19:48 +03:00
getCmdOpt ( $_SERVER [ 'argv' ], $aCMDOptions , $aResult , true , true );
2018-09-05 23:01:03 +03:00
if ( ! isset ( $aResult [ 'index-instances' ])) $aResult [ 'index-instances' ] = 1 ;
if ( ! isset ( $aResult [ 'index-rank' ])) $aResult [ 'index-rank' ] = 0 ;
2016-09-04 04:19:48 +03:00
date_default_timezone_set ( 'Etc/UTC' );
2019-02-24 18:14:36 +03:00
$oDB = new Nominatim\DB ();
$oDB -> connect ();
2016-09-04 04:19:48 +03:00
2019-02-24 18:14:36 +03:00
$aDSNInfo = Nominatim\DB :: parseDSN ( CONST_Database_DSN );
2018-09-05 23:01:03 +03:00
if ( ! isset ( $aDSNInfo [ 'port' ]) || ! $aDSNInfo [ 'port' ]) $aDSNInfo [ 'port' ] = 5432 ;
2016-09-04 04:19:48 +03:00
// cache memory to be used by osm2pgsql, should not be more than the available memory
2018-09-05 23:01:03 +03:00
$iCacheMemory = ( isset ( $aResult [ 'osm2pgsql-cache' ]) ? $aResult [ 'osm2pgsql-cache' ] : 2000 );
2016-09-08 04:16:22 +03:00
if ( $iCacheMemory + 500 > getTotalMemoryMB ()) {
2016-09-04 04:19:48 +03:00
$iCacheMemory = getCacheMemoryMB ();
echo " WARNING: resetting cache memory to $iCacheMemory\n " ;
}
2018-11-27 02:17:00 +03:00
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -S ' . CONST_Import_Style . ' -d ' . $aDSNInfo [ 'database' ] . ' -P ' . $aDSNInfo [ 'port' ];
2018-07-18 02:18:33 +03:00
if ( isset ( $aDSNInfo [ 'username' ]) && $aDSNInfo [ 'username' ]) {
$sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo [ 'username' ];
}
if ( isset ( $aDSNInfo [ 'hostspec' ]) && $aDSNInfo [ 'hostspec' ]) {
$sOsm2pgsqlCmd .= ' -H ' . $aDSNInfo [ 'hostspec' ];
}
2018-07-22 02:09:59 +03:00
$aProcEnv = null ;
2018-07-18 02:18:33 +03:00
if ( isset ( $aDSNInfo [ 'password' ]) && $aDSNInfo [ 'password' ]) {
2018-07-22 02:09:59 +03:00
$aProcEnv = array_merge ( array ( 'PGPASSWORD' => $aDSNInfo [ 'password' ]), $_ENV );
2018-07-18 02:18:33 +03:00
}
2018-02-12 22:43:39 +03:00
if ( ! is_null ( CONST_Osm2pgsql_Flatnode_File ) && CONST_Osm2pgsql_Flatnode_File ) {
2018-09-05 23:01:03 +03:00
$sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File ;
2016-09-04 04:19:48 +03:00
}
2017-05-25 17:26:09 +03:00
if ( $aResult [ 'init-updates' ]) {
2017-08-29 22:04:28 +03:00
// sanity check that the replication URL is correct
2018-09-05 23:01:03 +03:00
$sBaseState = file_get_contents ( CONST_Replication_Url . '/state.txt' );
2017-08-29 22:04:28 +03:00
if ( $sBaseState === false ) {
echo " \n Cannot find state.txt file at the configured replication URL. \n " ;
echo " Does the URL point to a directory containing OSM update data? \n \n " ;
2017-10-26 22:21:21 +03:00
fail ( 'replication URL not reachable.' );
2017-08-29 22:04:28 +03:00
}
2018-07-17 22:54:37 +03:00
// sanity check for pyosmium-get-changes
if ( ! CONST_Pyosmium_Binary ) {
echo " \n CONST_Pyosmium_Binary not configured. \n " ;
echo " You need to install pyosmium and set up the path to pyosmium-get-changes \n " ;
echo " in your local settings file. \n \n " ;
fail ( 'CONST_Pyosmium_Binary not configured' );
}
$aOutput = 0 ;
2018-09-05 23:01:03 +03:00
$sCmd = CONST_Pyosmium_Binary . ' --help' ;
2018-07-17 22:54:37 +03:00
exec ( $sCmd , $aOutput , $iRet );
if ( $iRet != 0 ) {
echo " Cannot execute pyosmium-get-changes. \n " ;
echo " Make sure you have pyosmium installed correctly \n " ;
echo " and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes. \n " ;
fail ( 'pyosmium-get-changes not found or not usable' );
}
2018-08-31 22:31:38 +03:00
2018-07-22 06:45:23 +03:00
if ( ! $aResult [ 'no-update-functions' ]) {
2018-09-05 23:01:03 +03:00
// instantiate setupClass to use the function therein
2018-10-02 23:59:10 +03:00
$cSetup = new SetupFunctions ( array (
'enable-diff-updates' => true ,
'verbose' => $aResult [ 'verbose' ]
));
2018-11-21 00:51:37 +03:00
$cSetup -> connect ();
2018-08-31 22:31:38 +03:00
$cSetup -> createFunctions ();
2017-05-25 17:26:09 +03:00
}
$sDatabaseDate = getDatabaseDate ( $oDB );
2019-02-24 18:14:36 +03:00
if ( ! $sDatabaseDate ) {
2017-10-26 22:21:21 +03:00
fail ( 'Cannot determine date of database.' );
2017-06-09 22:49:31 +03:00
}
2018-09-05 23:01:03 +03:00
$sWindBack = strftime ( '%Y-%m-%dT%H:%M:%SZ' , strtotime ( $sDatabaseDate ) - ( 3 * 60 * 60 ));
2017-05-25 17:26:09 +03:00
// get the appropriate state id
$aOutput = 0 ;
2018-09-05 23:01:03 +03:00
$sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url ;
2017-08-29 22:04:28 +03:00
exec ( $sCmd , $aOutput , $iRet );
if ( $iRet != 0 || $aOutput [ 0 ] == 'None' ) {
2017-05-25 17:26:09 +03:00
fail ( 'Error running pyosmium tools' );
}
2016-09-04 04:19:48 +03:00
2019-02-24 18:14:36 +03:00
$oDB -> exec ( 'TRUNCATE import_status' );
2018-07-21 06:06:09 +03:00
$sSQL = " INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES(' " ;
2018-09-05 23:01:03 +03:00
$sSQL .= $sDatabaseDate . " ', " . $aOutput [ 0 ] . ', true)' ;
2019-02-24 18:14:36 +03:00
try {
$oDB -> exec ( $sSQL );
} catch ( \Nominatim\DatabaseError $e ) {
2017-10-26 22:21:21 +03:00
fail ( 'Could not enter sequence into database.' );
2017-05-25 17:26:09 +03:00
}
echo " Done. Database updates will start at sequence $aOutput[0] ( $sWindBack ) \n " ;
}
2018-01-02 00:22:06 +03:00
if ( $aResult [ 'check-for-updates' ]) {
2019-03-10 17:42:58 +03:00
$aLastState = $oDB -> getRow ( 'SELECT sequence_id FROM import_status' );
2018-01-02 00:22:06 +03:00
if ( ! $aLastState [ 'sequence_id' ]) {
fail ( 'Updates not set up. Please run ./utils/update.php --init-updates.' );
}
2018-09-05 23:01:03 +03:00
system ( CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState [ 'sequence_id' ], $iRet );
2018-01-02 00:22:06 +03:00
exit ( $iRet );
}
2017-05-25 17:26:09 +03:00
if ( isset ( $aResult [ 'import-diff' ]) || isset ( $aResult [ 'import-file' ])) {
// import diffs and files directly (e.g. from osmosis --rri)
$sNextFile = isset ( $aResult [ 'import-diff' ]) ? $aResult [ 'import-diff' ] : $aResult [ 'import-file' ];
2017-10-03 13:07:26 +03:00
2016-09-08 04:16:22 +03:00
if ( ! file_exists ( $sNextFile )) {
2016-09-04 04:19:48 +03:00
fail ( " Cannot open $sNextFile\n " );
}
// Import the file
2018-09-05 23:01:03 +03:00
$sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile ;
echo $sCMD . " \n " ;
2018-07-22 02:09:59 +03:00
$iErrorLevel = runWithEnv ( $sCMD , $aProcEnv );
2016-09-04 04:19:48 +03:00
2016-09-08 04:16:22 +03:00
if ( $iErrorLevel ) {
2016-09-04 04:19:48 +03:00
fail ( " Error from osm2pgsql, $iErrorLevel\n " );
}
// Don't update the import status - we don't know what this file contains
}
2017-10-03 13:07:26 +03:00
if ( $aResult [ 'calculate-postcodes' ]) {
2017-10-26 22:21:21 +03:00
info ( 'Update postcodes centroids' );
2018-09-05 23:01:03 +03:00
$sTemplate = file_get_contents ( CONST_BasePath . '/sql/update-postcodes.sql' );
2017-10-03 13:07:26 +03:00
runSQLScript ( $sTemplate , true , true );
}
2018-09-05 23:01:03 +03:00
$sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc' ;
2016-09-04 04:19:48 +03:00
$bHaveDiff = false ;
$bUseOSMApi = isset ( $aResult [ 'import-from-main-api' ]) && $aResult [ 'import-from-main-api' ];
$sContentURL = '' ;
2016-09-08 04:16:22 +03:00
if ( isset ( $aResult [ 'import-node' ]) && $aResult [ 'import-node' ]) {
if ( $bUseOSMApi ) {
2018-09-05 23:01:03 +03:00
$sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult [ 'import-node' ];
2016-09-08 04:16:22 +03:00
} else {
2018-09-05 23:01:03 +03:00
$sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult [ 'import-node' ] . ');out%20meta;' ;
2016-09-04 04:19:48 +03:00
}
}
2016-09-08 04:16:22 +03:00
if ( isset ( $aResult [ 'import-way' ]) && $aResult [ 'import-way' ]) {
if ( $bUseOSMApi ) {
2018-09-05 23:01:03 +03:00
$sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult [ 'import-way' ] . '/full' ;
2016-09-08 04:16:22 +03:00
} else {
2018-09-05 23:01:03 +03:00
$sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult [ 'import-way' ] . ');node(w););out%20meta;' ;
2016-09-04 04:19:48 +03:00
}
}
2016-09-08 04:16:22 +03:00
if ( isset ( $aResult [ 'import-relation' ]) && $aResult [ 'import-relation' ]) {
if ( $bUseOSMApi ) {
2018-09-05 23:01:03 +03:00
$sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult [ 'import-relation' ] . '/full' ;
2016-09-08 04:16:22 +03:00
} else {
2018-09-05 23:01:03 +03:00
$sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult [ 'import-relation' ] . ');way(r);node(w));node(r));out%20meta;' ;
2016-09-04 04:19:48 +03:00
}
}
2016-09-08 04:16:22 +03:00
if ( $sContentURL ) {
2017-05-25 17:26:09 +03:00
file_put_contents ( $sTemporaryFile , file_get_contents ( $sContentURL ));
2016-09-04 04:19:48 +03:00
$bHaveDiff = true ;
}
2016-09-08 04:16:22 +03:00
if ( $bHaveDiff ) {
2016-09-04 04:19:48 +03:00
// import generated change file
2018-09-05 23:01:03 +03:00
$sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile ;
echo $sCMD . " \n " ;
2018-07-22 02:09:59 +03:00
$iErrorLevel = runWithEnv ( $sCMD , $aProcEnv );
2016-09-08 04:16:22 +03:00
if ( $iErrorLevel ) {
2016-09-04 04:19:48 +03:00
fail ( " osm2pgsql exited with error level $iErrorLevel\n " );
}
}
2016-09-08 04:16:22 +03:00
if ( $aResult [ 'deduplicate' ]) {
2019-02-24 18:14:36 +03:00
$oDB = new Nominatim\DB ();
$oDB -> connect ();
2017-04-04 19:09:28 +03:00
2019-02-24 18:14:36 +03:00
if ( $oDB -> getPostgresVersion () < 9.3 ) {
2017-10-26 22:21:21 +03:00
fail ( 'ERROR: deduplicate is only currently supported in postgresql 9.3' );
2016-09-04 04:19:48 +03:00
}
$sSQL = 'select partition from country_name order by country_code' ;
2019-03-10 17:42:58 +03:00
$aPartitions = $oDB -> getCol ( $sSQL );
2016-09-04 04:19:48 +03:00
$aPartitions [] = 0 ;
2017-05-25 17:26:09 +03:00
// we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
2017-04-04 20:00:36 +03:00
foreach ( $aPartitions as $i => $sPartition ) {
2018-09-05 23:01:03 +03:00
$sSQL = 'select count(*) from search_name_' . $sPartition ;
2019-03-10 17:42:58 +03:00
$nEntries = $oDB -> getOne ( $sSQL );
2017-04-04 20:00:36 +03:00
if ( $nEntries == 0 ) {
unset ( $aPartitions [ $i ]);
}
}
2016-10-14 01:01:16 +03:00
$sSQL = " select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' " ;
2017-10-26 22:21:21 +03:00
$sSQL .= ' and class is null and type is null and country_code is null' ;
$sSQL .= ' group by word_token having count(*) > 1 order by word_token' ;
2019-03-10 17:42:58 +03:00
$aDuplicateTokens = $oDB -> getAll ( $sSQL );
2016-09-08 04:16:22 +03:00
foreach ( $aDuplicateTokens as $aToken ) {
2018-09-05 23:01:03 +03:00
if ( trim ( $aToken [ 'word_token' ]) == '' || trim ( $aToken [ 'word_token' ]) == '-' ) continue ;
echo 'Deduping ' . $aToken [ 'word_token' ] . " \n " ;
2017-10-26 22:21:21 +03:00
$sSQL = 'select word_id,' ;
$sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num' ;
2018-09-05 23:01:03 +03:00
$sSQL .= " from word where word_token = ' " . $aToken [ 'word_token' ];
2016-10-14 01:01:16 +03:00
$sSQL .= " ' and class is null and type is null and country_code is null order by num desc " ;
2019-03-10 17:42:58 +03:00
$aTokenSet = $oDB -> getAll ( $sSQL );
2016-09-04 04:19:48 +03:00
$aKeep = array_shift ( $aTokenSet );
$iKeepID = $aKeep [ 'word_id' ];
2016-09-08 04:16:22 +03:00
foreach ( $aTokenSet as $aRemove ) {
2017-10-26 22:21:21 +03:00
$sSQL = 'update search_name set' ;
2018-09-05 23:01:03 +03:00
$sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove [ 'word_id' ] . ',' . $iKeepID . '),' ;
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove [ 'word_id' ] . ',' . $iKeepID . ')' ;
$sSQL .= ' where name_vector @> ARRAY[' . $aRemove [ 'word_id' ] . ']' ;
2019-03-10 17:42:58 +03:00
$oDB -> exec ( $sSQL );
2016-09-04 04:19:48 +03:00
2017-10-26 22:21:21 +03:00
$sSQL = 'update search_name set' ;
2018-09-05 23:01:03 +03:00
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove [ 'word_id' ] . ',' . $iKeepID . ')' ;
$sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove [ 'word_id' ] . ']' ;
2019-03-10 17:42:58 +03:00
$oDB -> exec ( $sSQL );
2016-09-04 04:19:48 +03:00
2017-10-26 22:21:21 +03:00
$sSQL = 'update location_area_country set' ;
2018-09-05 23:01:03 +03:00
$sSQL .= ' keywords = array_replace(keywords,' . $aRemove [ 'word_id' ] . ',' . $iKeepID . ')' ;
$sSQL .= ' where keywords @> ARRAY[' . $aRemove [ 'word_id' ] . ']' ;
2019-03-10 17:42:58 +03:00
$oDB -> exec ( $sSQL );
2016-09-04 04:19:48 +03:00
2016-09-08 04:16:22 +03:00
foreach ( $aPartitions as $sPartition ) {
2018-09-05 23:01:03 +03:00
$sSQL = 'update search_name_' . $sPartition . ' set' ;
$sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove [ 'word_id' ] . ',' . $iKeepID . ')' ;
$sSQL .= ' where name_vector @> ARRAY[' . $aRemove [ 'word_id' ] . ']' ;
2019-03-10 17:42:58 +03:00
$oDB -> exec ( $sSQL );
2016-09-04 04:19:48 +03:00
2017-10-26 22:21:21 +03:00
$sSQL = 'update location_area_country set' ;
2018-09-05 23:01:03 +03:00
$sSQL .= ' keywords = array_replace(keywords,' . $aRemove [ 'word_id' ] . ',' . $iKeepID . ')' ;
$sSQL .= ' where keywords @> ARRAY[' . $aRemove [ 'word_id' ] . ']' ;
2019-03-10 17:42:58 +03:00
$oDB -> exec ( $sSQL );
2016-09-04 04:19:48 +03:00
}
2018-09-05 23:01:03 +03:00
$sSQL = 'delete from word where word_id = ' . $aRemove [ 'word_id' ];
2019-03-10 17:42:58 +03:00
$oDB -> exec ( $sSQL );
2016-09-04 04:19:48 +03:00
}
}
}
2017-10-28 19:34:22 +03:00
if ( $aResult [ 'recompute-word-counts' ]) {
info ( 'Recompute frequency of full-word search terms' );
2018-09-05 23:01:03 +03:00
$sTemplate = file_get_contents ( CONST_BasePath . '/sql/words_from_search_name.sql' );
2017-10-28 19:34:22 +03:00
runSQLScript ( $sTemplate , true , true );
}
2016-09-08 04:16:22 +03:00
if ( $aResult [ 'index' ]) {
2018-09-05 23:01:03 +03:00
$sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo [ 'database' ] . ' -P ' . $aDSNInfo [ 'port' ] . ' -t ' . $aResult [ 'index-instances' ] . ' -r ' . $aResult [ 'index-rank' ];
2018-07-18 02:18:33 +03:00
if ( isset ( $aDSNInfo [ 'hostspec' ]) && $aDSNInfo [ 'hostspec' ]) {
2018-07-22 02:09:59 +03:00
$sCmd .= ' -H ' . $aDSNInfo [ 'hostspec' ];
2018-07-18 02:18:33 +03:00
}
if ( isset ( $aDSNInfo [ 'username' ]) && $aDSNInfo [ 'username' ]) {
2018-07-22 02:09:59 +03:00
$sCmd .= ' -U ' . $aDSNInfo [ 'username' ];
2018-07-18 02:18:33 +03:00
}
2018-07-22 02:09:59 +03:00
runWithEnv ( $sCmd , $aProcEnv );
2019-01-26 17:01:39 +03:00
2019-02-24 18:14:36 +03:00
$oDB -> exec ( 'update import_status set indexed = true' );
2016-09-04 04:19:48 +03:00
}
2018-11-24 12:20:24 +03:00
if ( $aResult [ 'update-address-levels' ]) {
echo 'Updating address levels from ' . CONST_Address_Level_Config . " . \n " ;
$oAlParser = new \Nominatim\Setup\AddressLevelParser ( CONST_Address_Level_Config );
$oAlParser -> createTable ( $oDB , 'address_levels' );
}
2019-10-28 23:17:42 +03:00
if ( $aResult [ 'recompute-importance' ]) {
echo 'Updating importance values for database.\n' ;
$oDB = new Nominatim\DB ();
$oDB -> connect ();
$sSQL = 'ALTER TABLE placex DISABLE TRIGGER ALL;' ;
$sSQL .= 'UPDATE placex SET (wikipedia, importance) = (SELECT wikipedia, importance FROM compute_importance(extratags, country_code, osm_type, osm_id));' ;
2019-10-31 20:26:41 +03:00
$sSQL .= 'UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance FROM placex d WHERE s.place_id = d.linked_place_id and d.wikipedia is not null and (s.wikipedia is null or s.importance < d.importance);' ;
2019-10-28 23:17:42 +03:00
$sSQL .= 'ALTER TABLE placex ENABLE TRIGGER ALL;' ;
$oDB -> exec ( $sSQL );
}
2016-09-08 04:16:22 +03:00
if ( $aResult [ 'import-osmosis' ] || $aResult [ 'import-osmosis-all' ]) {
//
2016-09-04 04:19:48 +03:00
if ( strpos ( CONST_Replication_Url , 'download.geofabrik.de' ) !== false && CONST_Replication_Update_Interval < 86400 ) {
2018-02-08 19:52:26 +03:00
fail ( 'Error: Update interval too low for download.geofabrik.de. ' .
2018-09-05 23:01:03 +03:00
" Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process) \n " );
2016-09-04 04:19:48 +03:00
}
2018-09-05 23:01:03 +03:00
$sImportFile = CONST_InstallPath . '/osmosischange.osc' ;
$sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size ;
$sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile ;
$sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo [ 'database' ] . ' -P ' . $aDSNInfo [ 'port' ] . ' -t ' . $aResult [ 'index-instances' ];
2018-07-18 02:18:33 +03:00
if ( isset ( $aDSNInfo [ 'hostspec' ]) && $aDSNInfo [ 'hostspec' ]) {
$sCMDIndex .= ' -H ' . $aDSNInfo [ 'hostspec' ];
}
if ( isset ( $aDSNInfo [ 'username' ]) && $aDSNInfo [ 'username' ]) {
$sCMDIndex .= ' -U ' . $aDSNInfo [ 'username' ];
}
2016-09-04 04:19:48 +03:00
2016-09-08 04:16:22 +03:00
while ( true ) {
2016-09-04 04:19:48 +03:00
$fStartTime = time ();
2019-03-10 17:42:58 +03:00
$aLastState = $oDB -> getRow ( 'SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status' );
2017-05-25 17:26:09 +03:00
if ( ! $aLastState [ 'sequence_id' ]) {
echo " Updates not set up. Please run ./utils/update.php --init-updates. \n " ;
exit ( 1 );
}
2018-09-05 23:01:03 +03:00
echo 'Currently at sequence ' . $aLastState [ 'sequence_id' ] . ' (' . $aLastState [ 'lastimportdate' ] . ') - ' . $aLastState [ 'indexed' ] . " indexed \n " ;
2017-05-25 17:26:09 +03:00
$sBatchEnd = $aLastState [ 'lastimportdate' ];
$iEndSequence = $aLastState [ 'sequence_id' ];
2019-04-02 17:52:37 +03:00
if ( $aLastState [ 'indexed' ]) {
2017-05-25 17:26:09 +03:00
// Sleep if the update interval has not yet been reached.
2017-06-11 10:05:24 +03:00
$fNextUpdate = $aLastState [ 'unix_ts' ] + CONST_Replication_Update_Interval ;
2017-05-25 17:26:09 +03:00
if ( $fNextUpdate > $fStartTime ) {
$iSleepTime = $fNextUpdate - $fStartTime ;
echo " Waiting for next update for $iSleepTime sec. " ;
sleep ( $iSleepTime );
}
// Download the next batch of changes.
do {
$fCMDStartTime = time ();
2017-05-27 18:02:34 +03:00
$iNextSeq = ( int ) $aLastState [ 'sequence_id' ];
2017-05-25 17:26:09 +03:00
unset ( $aOutput );
echo " $sCMDDownload -I $iNextSeq\n " ;
2018-01-31 02:38:05 +03:00
if ( file_exists ( $sImportFile )) {
unlink ( $sImportFile );
}
2018-09-05 23:01:03 +03:00
exec ( $sCMDDownload . ' -I ' . $iNextSeq , $aOutput , $iResult );
2017-05-25 17:26:09 +03:00
if ( $iResult == 3 ) {
2018-09-05 23:01:03 +03:00
echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec. \n " ;
2016-09-04 04:19:48 +03:00
sleep ( CONST_Replication_Recheck_Interval );
2017-10-08 18:00:59 +03:00
} elseif ( $iResult != 0 ) {
2017-05-25 17:26:09 +03:00
echo 'ERROR: updates failed.' ;
exit ( $iResult );
} else {
2018-09-05 23:01:03 +03:00
$iEndSequence = ( int ) $aOutput [ 0 ];
2016-09-04 04:19:48 +03:00
}
2017-05-25 17:26:09 +03:00
} while ( $iResult );
2017-06-11 10:05:24 +03:00
// get the newest object from the diff file
$sBatchEnd = 0 ;
$iRet = 0 ;
2018-09-05 23:01:03 +03:00
exec ( CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile , $sBatchEnd , $iRet );
2017-06-11 10:05:24 +03:00
if ( $iRet == 5 ) {
echo " Diff file is empty. skipping import. \n " ;
if ( ! $aResult [ 'import-osmosis-all' ]) {
exit ( 0 );
} else {
continue ;
}
}
if ( $iRet != 0 ) {
fail ( 'Error getting date from diff file.' );
}
$sBatchEnd = $sBatchEnd [ 0 ];
2017-05-25 17:26:09 +03:00
// Import the file
2016-09-04 04:19:48 +03:00
$fCMDStartTime = time ();
2018-09-05 23:01:03 +03:00
echo $sCMDImport . " \n " ;
2017-05-25 17:26:09 +03:00
unset ( $sJunk );
2018-07-22 02:09:59 +03:00
$iErrorLevel = runWithEnv ( $sCMDImport , $aProcEnv );
2017-05-25 17:26:09 +03:00
if ( $iErrorLevel ) {
echo " Error executing osm2pgsql: $iErrorLevel\n " ;
exit ( $iErrorLevel );
2016-09-04 04:19:48 +03:00
}
2017-05-25 17:26:09 +03:00
// write the update logs
2016-09-04 04:19:48 +03:00
$iFileSize = filesize ( $sImportFile );
2017-10-08 18:00:59 +03:00
$sSQL = 'INSERT INTO import_osmosis_log' ;
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)' ;
$sSQL .= " values (' $sBatchEnd ', $iEndSequence , $iFileSize ,' " ;
2018-09-05 23:01:03 +03:00
$sSQL .= date ( 'Y-m-d H:i:s' , $fCMDStartTime ) . " ',' " ;
$sSQL .= date ( 'Y-m-d H:i:s' ) . " ','import') " ;
2016-09-04 04:19:48 +03:00
var_Dump ( $sSQL );
2019-03-10 17:42:58 +03:00
$oDB -> exec ( $sSQL );
2016-09-04 04:19:48 +03:00
2017-05-25 17:26:09 +03:00
// update the status
$sSQL = " UPDATE import_status SET lastimportdate = ' $sBatchEnd ', indexed=false, sequence_id = $iEndSequence " ;
var_Dump ( $sSQL );
2019-03-10 17:42:58 +03:00
$oDB -> exec ( $sSQL );
2018-09-05 23:01:03 +03:00
echo date ( 'Y-m-d H:i:s' ) . " Completed download step for $sBatchEnd in " . round (( time () - $fCMDStartTime ) / 60 , 2 ) . " minutes \n " ;
2016-09-04 04:19:48 +03:00
}
// Index file
2016-09-08 04:16:22 +03:00
if ( ! $aResult [ 'no-index' ]) {
2017-05-25 17:26:09 +03:00
$sThisIndexCmd = $sCMDIndex ;
$fCMDStartTime = time ();
2016-09-04 04:19:48 +03:00
echo " $sThisIndexCmd\n " ;
2018-07-22 02:09:59 +03:00
$iErrorLevel = runWithEnv ( $sThisIndexCmd , $aProcEnv );
2016-09-08 04:16:22 +03:00
if ( $iErrorLevel ) {
2016-09-04 04:19:48 +03:00
echo " Error: $iErrorLevel\n " ;
exit ( $iErrorLevel );
}
2017-10-08 18:00:59 +03:00
$sSQL = 'INSERT INTO import_osmosis_log' ;
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)' ;
2019-01-04 23:51:18 +03:00
$sSQL .= " values (' $sBatchEnd ', $iEndSequence ,NULL,' " ;
2018-09-05 23:01:03 +03:00
$sSQL .= date ( 'Y-m-d H:i:s' , $fCMDStartTime ) . " ',' " ;
$sSQL .= date ( 'Y-m-d H:i:s' ) . " ','index') " ;
2017-05-25 17:26:09 +03:00
var_Dump ( $sSQL );
2019-02-24 18:14:36 +03:00
$oDB -> exec ( $sSQL );
2018-09-05 23:01:03 +03:00
echo date ( 'Y-m-d H:i:s' ) . " Completed index step for $sBatchEnd in " . round (( time () - $fCMDStartTime ) / 60 , 2 ) . " minutes \n " ;
2016-09-04 04:19:48 +03:00
2018-07-21 06:06:09 +03:00
$sSQL = 'update import_status set indexed = true' ;
2019-02-24 18:14:36 +03:00
$oDB -> exec ( $sSQL );
2019-01-26 17:25:00 +03:00
} else {
if ( $aResult [ 'import-osmosis-all' ]) {
echo " Error: --no-index cannot be used with continuous imports (--import-osmosis-all). \n " ;
exit ( 1 );
}
2017-05-25 17:26:09 +03:00
}
2016-09-04 04:19:48 +03:00
$fDuration = time () - $fStartTime ;
2018-09-05 23:01:03 +03:00
echo date ( 'Y-m-d H:i:s' ) . " Completed all for $sBatchEnd in " . round ( $fDuration / 60 , 2 ) . " minutes \n " ;
if ( ! $aResult [ 'import-osmosis-all' ]) exit ( 0 );
2016-09-04 04:19:48 +03:00
}
}