From 68beec5590b917167271accdc6b1244836eb44f6 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 16 Apr 2021 17:09:40 +0200 Subject: [PATCH 1/3] remove installation of PHP util scripts --- CMakeLists.txt | 15 --------------- cmake/script.tmpl | 14 -------------- 2 files changed, 29 deletions(-) delete mode 100755 cmake/script.tmpl diff --git a/CMakeLists.txt b/CMakeLists.txt index f3247900..fa92dd02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,21 +109,6 @@ if (BUILD_IMPORTER) " wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz") endif() - set(CUSTOMSCRIPTS - check_import_finished.php - country_languages.php - export.php - query.php - setup.php - update.php - warm.php - ) - - foreach (script_source ${CUSTOMSCRIPTS}) - configure_file(${PROJECT_SOURCE_DIR}/cmake/script.tmpl - ${PROJECT_BINARY_DIR}/utils/${script_source}) - endforeach() - configure_file(${PROJECT_SOURCE_DIR}/cmake/tool.tmpl ${PROJECT_BINARY_DIR}/nominatim) endif() diff --git a/cmake/script.tmpl b/cmake/script.tmpl deleted file mode 100755 index 3fbe535e..00000000 --- a/cmake/script.tmpl +++ /dev/null @@ -1,14 +0,0 @@ -#!@PHP_BIN@ -Cq - Date: Fri, 16 Apr 2021 17:28:51 +0200 Subject: [PATCH 2/3] remove PHP code for transition functions --- lib-php/admin/check_import_finished.php | 10 - lib-php/admin/setup.php | 218 -------------------- lib-php/admin/specialphrases.php | 11 - lib-php/admin/update.php | 88 -------- lib-php/cmd.php | 52 ----- lib-php/setup/SetupClass.php | 261 ------------------------ lib-php/setup_functions.php | 15 -- 7 files changed, 655 deletions(-) delete mode 100644 lib-php/admin/check_import_finished.php delete mode 100644 lib-php/admin/setup.php delete mode 100644 lib-php/admin/specialphrases.php delete mode 100755 lib-php/setup/SetupClass.php diff --git a/lib-php/admin/check_import_finished.php b/lib-php/admin/check_import_finished.php deleted file mode 100644 index d5d011c4..00000000 --- a/lib-php/admin/check_import_finished.php +++ /dev/null @@ -1,10 +0,0 @@ -addParams('admin', '--check-database') - ->run(); diff --git a/lib-php/admin/setup.php b/lib-php/admin/setup.php deleted file mode 100644 index 7523527a..00000000 --- a/lib-php/admin/setup.php +++ /dev/null @@ -1,218 +0,0 @@ -addParams('--threads', $iInstances); - if ($aCMDResult['ignore-errors'] ?? false) { - $oCmd->addParams('--ignore-errors'); - } - if ($aCMDResult['quiet'] ?? false) { - $oCmd->addParams('--quiet'); - } - if ($aCMDResult['verbose'] ?? false) { - $oCmd->addParams('--verbose'); - } - $oCmd->run(true); -} - - -//******************************************************* -// Making some sanity check: -// Check if osm-file is set and points to a valid file -if ($aCMDResult['import-data'] || $aCMDResult['all']) { - // to remain in /lib/setup_functions.php function - checkInFile($aCMDResult['osm-file']); -} - -// ****************************************************** -// instantiate Setup class -$oSetup = new SetupFunctions($aCMDResult); - -// ******************************************************* -// go through complete process if 'all' is selected or start selected functions -if ($aCMDResult['create-db'] || $aCMDResult['all']) { - $bDidSomething = true; - run((clone($oNominatimCmd))->addParams('transition', '--create-db')); -} - -if ($aCMDResult['setup-db'] || $aCMDResult['all']) { - $bDidSomething = true; - $oCmd = (clone($oNominatimCmd))->addParams('transition', '--setup-db'); - - if ($aCMDResult['no-partitions'] ?? false) { - $oCmd->addParams('--no-partitions'); - } - - run($oCmd); -} - -if ($aCMDResult['import-data'] || $aCMDResult['all']) { - $bDidSomething = true; - $oCmd = (clone($oNominatimCmd)) - ->addParams('transition', '--import-data') - ->addParams('--osm-file', $aCMDResult['osm-file']); - if ($aCMDResult['drop'] ?? false) { - $oCmd->addParams('--drop'); - } - - run($oCmd); -} - -if ($aCMDResult['create-functions'] || $aCMDResult['all']) { - $bDidSomething = true; - $oSetup->createSqlFunctions(); -} - -if ($aCMDResult['create-tables'] || $aCMDResult['all']) { - $bDidSomething = true; - $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-tables'); - - if ($aCMDResult['reverse-only'] ?? false) { - $oCmd->addParams('--reverse-only'); - } - - run($oCmd); -} - -if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { - $bDidSomething = true; - run((clone($oNominatimCmd))->addParams('transition', '--create-partition-tables')); -} - -if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { - $bDidSomething = true; - $oSetup->createSqlFunctions(); // also create partition functions -} - -if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { - $bDidSomething = true; - // ignore errors! - (clone($oNominatimCmd))->addParams('refresh', '--wiki-data')->run(); -} - -if ($aCMDResult['load-data'] || $aCMDResult['all']) { - $bDidSomething = true; - run((clone($oNominatimCmd))->addParams('transition', '--load-data')); -} - -if ($aCMDResult['import-tiger-data']) { - $bDidSomething = true; - $sTigerPath = getSetting('TIGER_DATA_PATH', CONST_InstallDir.'/tiger'); - run((clone($oNominatimCmd))->addParams('transition', '--tiger-data', $sTigerPath)); -} - -if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { - $bDidSomething = true; - $oSetup->calculatePostcodes($aCMDResult['all']); -} - -if ($aCMDResult['index'] || $aCMDResult['all']) { - $bDidSomething = true; - $oCmd = (clone($oNominatimCmd))->addParams('transition', '--index'); - if ($aCMDResult['index-noanalyse'] ?? false) { - $oCmd->addParams('--no-analyse'); - } - - run($oCmd); -} - -if ($aCMDResult['drop']) { - $bDidSomething = true; - run((clone($oNominatimCmd))->addParams('freeze')); -} - -if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { - $bDidSomething = true; - - $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-search-indices'); - - if ($aCMDResult['drop'] ?? false) { - $oCmd->addParams('--drop'); - } - - run($oCmd); -} - -if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { - $bDidSomething = true; - run(clone($oNominatimCmd))->addParams('transition', '--create-country-names'); -} - -if ($aCMDResult['setup-website'] || $aCMDResult['all']) { - $bDidSomething = true; - run((clone($oNominatimCmd))->addParams('refresh', '--website')); -} - -// ****************************************************** -// If we did something, repeat the warnings -if (!$bDidSomething) { - showUsage($aCMDOptions, true); -} else { - echo "Summary of warnings:\n\n"; - repeatWarnings(); - echo "\n"; - info('Setup finished.'); -} diff --git a/lib-php/admin/specialphrases.php b/lib-php/admin/specialphrases.php deleted file mode 100644 index 84bcfb5c..00000000 --- a/lib-php/admin/specialphrases.php +++ /dev/null @@ -1,11 +0,0 @@ - -addParams('special-phrases', '--import-from-wiki') - ->run(); diff --git a/lib-php/admin/update.php b/lib-php/admin/update.php index 81e0b832..ea58f37c 100644 --- a/lib-php/admin/update.php +++ b/lib-php/admin/update.php @@ -3,12 +3,9 @@ require_once(CONST_LibDir.'/init-cmd.php'); require_once(CONST_LibDir.'/setup_functions.php'); -require_once(CONST_LibDir.'/setup/SetupClass.php'); ini_set('memory_limit', '800M'); -use Nominatim\Setup\SetupFunctions as SetupFunctions; - // (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) $aCMDOptions = array( @@ -17,13 +14,6 @@ $aCMDOptions array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), - array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), - array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), - array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), - array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), - array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), @@ -35,14 +25,6 @@ $aCMDOptions array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index'), - array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), - array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), - - array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), - array('update-address-levels', '', 0, 1, 0, 0, 'bool', 'Reimport address level configuration (EXPERT)'), - array('recompute-importance', '', 0, 1, 0, 0, 'bool', 'Recompute place importances'), - array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'), ); @@ -51,9 +33,6 @@ getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true); loadSettings($aCMDResult['project-dir'] ?? getcwd()); setupHTTPProxy(); -if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1; -if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0; - date_default_timezone_set('Etc/UTC'); $oDB = new Nominatim\DB(); @@ -103,35 +82,6 @@ if ($fPostgresVersion >= 11.0) { ); } -$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')); - -function run($oCmd) -{ - global $aCMDResult; - if ($aCMDResult['quiet'] ?? false) { - $oCmd->addParams('--quiet'); - } - if ($aCMDResult['verbose'] ?? false) { - $oCmd->addParams('--verbose'); - } - $oCmd->run(true); -} - - -if ($aResult['init-updates']) { - $oCmd = (clone($oNominatimCmd))->addParams('replication', '--init'); - - if ($aResult['no-update-functions']) { - $oCmd->addParams('--no-update-functions'); - } - - run($oCmd); -} - -if ($aResult['check-for-updates']) { - exit((clone($oNominatimCmd))->addParams('replication', '--check-for-updates')->run()); -} - if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { // import diffs and files directly (e.g. from osmosis --rri) $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file']; @@ -152,10 +102,6 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { // Don't update the import status - we don't know what this file contains } -if ($aResult['calculate-postcodes']) { - run((clone($oNominatimCmd))->addParams('refresh', '--postcodes')); -} - $sTemporaryFile = CONST_InstallDir.'/osmosischange.osc'; $bHaveDiff = false; $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api']; @@ -200,37 +146,3 @@ if ($bHaveDiff) { fail("osm2pgsql exited with error level $iRet\n"); } } - -if ($aResult['recompute-word-counts']) { - run((clone($oNominatimCmd))->addParams('refresh', '--word-counts')); -} - -if ($aResult['index']) { - run((clone $oNominatimCmd) - ->addParams('index', '--minrank', $aResult['index-rank']) - ->addParams('--threads', $aResult['index-instances'])); -} - -if ($aResult['update-address-levels']) { - run((clone($oNominatimCmd))->addParams('refresh', '--address-levels')); -} - -if ($aResult['recompute-importance']) { - run((clone($oNominatimCmd))->addParams('refresh', '--importance')); -} - -if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { - $oCmd = (clone($oNominatimCmd)) - ->addParams('replication') - ->addParams('--threads', $aResult['index-instances']); - - if (!$aResult['import-osmosis-all']) { - $oCmd->addParams('--once'); - } - - if ($aResult['no-index']) { - $oCmd->addParams('--no-index'); - } - - run($oCmd); -} diff --git a/lib-php/cmd.php b/lib-php/cmd.php index 5a12f99a..9c971e5f 100644 --- a/lib-php/cmd.php +++ b/lib-php/cmd.php @@ -144,58 +144,6 @@ function repeatWarnings() } -function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreErrors = false) -{ - // Convert database DSN to psql parameters - $aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN')); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - - $oCmd = new \Nominatim\Shell('psql'); - $oCmd->addParams('--port', $aDSNInfo['port']); - $oCmd->addParams('--dbname', $aDSNInfo['database']); - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $oCmd->addParams('--host', $aDSNInfo['hostspec']); - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $oCmd->addParams('--username', $aDSNInfo['username']); - } - if (isset($aDSNInfo['password'])) { - $oCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']); - } - if (!$bVerbose) { - $oCmd->addParams('--quiet'); - } - if ($bfatal && !$bIgnoreErrors) { - $oCmd->addParams('-v', 'ON_ERROR_STOP=1'); - } - - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => STDOUT, - 2 => STDERR - ); - $ahPipes = null; - $hProcess = @proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv); - if (!is_resource($hProcess)) { - fail('unable to start pgsql'); - } - - if (!$bVerbose) { - fwrite($ahPipes[0], 'set client_min_messages to WARNING;'); - } - - while (strlen($sScript)) { - $iWritten = fwrite($ahPipes[0], $sScript); - if ($iWritten <= 0) break; - $sScript = substr($sScript, $iWritten); - } - fclose($ahPipes[0]); - $iReturn = proc_close($hProcess); - if ($bfatal && $iReturn > 0) { - fail("pgsql returned with error code ($iReturn)"); - } -} - function setupHTTPProxy() { if (!getSettingBool('HTTP_PROXY')) { diff --git a/lib-php/setup/SetupClass.php b/lib-php/setup/SetupClass.php deleted file mode 100755 index d07adce7..00000000 --- a/lib-php/setup/SetupClass.php +++ /dev/null @@ -1,261 +0,0 @@ -iInstances = isset($aCMDResult['threads']) - ? $aCMDResult['threads'] - : (min(16, getProcessorCount()) - 1); - - if ($this->iInstances < 1) { - $this->iInstances = 1; - warn('resetting threads to '.$this->iInstances); - } - - // parse database string - $this->aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN')); - if (!isset($this->aDSNInfo['port'])) { - $this->aDSNInfo['port'] = 5432; - } - - // setting member variables based on command line options stored in $aCMDResult - $this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet']; - $this->bVerbose = $aCMDResult['verbose']; - - //setting default values which are not set by the update.php array - if (isset($aCMDResult['ignore-errors'])) { - $this->sIgnoreErrors = $aCMDResult['ignore-errors']; - } else { - $this->sIgnoreErrors = false; - } - if (isset($aCMDResult['enable-debug-statements'])) { - $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; - } else { - $this->bEnableDebugStatements = false; - } - if (isset($aCMDResult['enable-diff-updates'])) { - $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; - } else { - $this->bEnableDiffUpdates = false; - } - - $this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop']; - - $this->oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')); - if ($this->bQuiet) { - $this->oNominatimCmd->addParams('--quiet'); - } - if ($this->bVerbose) { - $this->oNominatimCmd->addParams('--verbose'); - } - } - - public function calculatePostcodes($bCMDResultAll) - { - info('Calculate Postcodes'); - $this->pgsqlRunScriptFile(CONST_SqlDir.'/postcode_tables.sql'); - - $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz'; - if (file_exists($sPostcodeFilename)) { - $this->pgsqlRunScriptFile($sPostcodeFilename); - } else { - warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.'); - } - - $sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz'; - if (file_exists($sPostcodeFilename)) { - $this->pgsqlRunScriptFile($sPostcodeFilename); - } else { - warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.'); - } - - - $this->db()->exec('TRUNCATE location_postcode'); - - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; - $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; - $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; - $sSQL .= ' FROM placex'; - $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; - $sSQL .= ' AND geometry IS NOT null'; - $sSQL .= ' GROUP BY country_code, pc'; - $this->db()->exec($sSQL); - - // only add postcodes that are not yet available in OSM - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; - $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; - $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'us')"; - $this->db()->exec($sSQL); - - // add missing postcodes for GB (if available) - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; - $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'gb')"; - $this->db()->exec($sSQL); - - if (!$bCMDResultAll) { - $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; - $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; - $this->db()->exec($sSQL); - } - - $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; - $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; - $this->db()->exec($sSQL); - } - - /** - * Return the connection to the database. - * - * @return Database object. - * - * Creates a new connection if none exists yet. Otherwise reuses the - * already established connection. - */ - private function db() - { - if (is_null($this->oDB)) { - $this->oDB = new \Nominatim\DB(); - $this->oDB->connect(); - } - - return $this->oDB; - } - - private function pgsqlRunScript($sScript, $bfatal = true) - { - runSQLScript( - $sScript, - $bfatal, - $this->bVerbose, - $this->sIgnoreErrors - ); - } - - public function createSqlFunctions() - { - $oCmd = (clone($this->oNominatimCmd)) - ->addParams('refresh', '--functions'); - - if (!$this->bEnableDiffUpdates) { - $oCmd->addParams('--no-diff-updates'); - } - - if ($this->bEnableDebugStatements) { - $oCmd->addParams('--enable-debug-statements'); - } - - $oCmd->run(!$this->sIgnoreErrors); - } - - private function pgsqlRunScriptFile($sFilename) - { - if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - - $oCmd = (new \Nominatim\Shell('psql')) - ->addParams('--port', $this->aDSNInfo['port']) - ->addParams('--dbname', $this->aDSNInfo['database']); - - if (!$this->bVerbose) { - $oCmd->addParams('--quiet'); - } - if (isset($this->aDSNInfo['hostspec'])) { - $oCmd->addParams('--host', $this->aDSNInfo['hostspec']); - } - if (isset($this->aDSNInfo['username'])) { - $oCmd->addParams('--username', $this->aDSNInfo['username']); - } - if (isset($this->aDSNInfo['password'])) { - $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); - } - $ahGzipPipes = null; - if (preg_match('/\\.gz$/', $sFilename)) { - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $oZcatCmd = new \Nominatim\Shell('zcat', $sFilename); - - $hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes); - if (!is_resource($hGzipProcess)) fail('unable to start zcat'); - $aReadPipe = $ahGzipPipes[1]; - fclose($ahGzipPipes[0]); - } else { - $oCmd->addParams('--file', $sFilename); - $aReadPipe = array('pipe', 'r'); - } - $aDescriptors = array( - 0 => $aReadPipe, - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - - $hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv); - if (!is_resource($hProcess)) fail('unable to start pgsql'); - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - $iReturn = proc_close($hProcess); - if ($iReturn > 0) { - fail("pgsql returned with error code ($iReturn)"); - } - if ($ahGzipPipes) { - fclose($ahGzipPipes[1]); - proc_close($hGzipProcess); - } - } - - private function replaceSqlPatterns($sSql) - { - $sSql = str_replace('{www-user}', getSetting('DATABASE_WEBUSER'), $sSql); - - $aPatterns = array( - '{ts:address-data}' => getSetting('TABLESPACE_ADDRESS_DATA'), - '{ts:address-index}' => getSetting('TABLESPACE_ADDRESS_INDEX'), - '{ts:search-data}' => getSetting('TABLESPACE_SEARCH_DATA'), - '{ts:search-index}' => getSetting('TABLESPACE_SEARCH_INDEX'), - '{ts:aux-data}' => getSetting('TABLESPACE_AUX_DATA'), - '{ts:aux-index}' => getSetting('TABLESPACE_AUX_INDEX') - ); - - foreach ($aPatterns as $sPattern => $sTablespace) { - if ($sTablespace) { - $sSql = str_replace($sPattern, 'TABLESPACE "'.$sTablespace.'"', $sSql); - } else { - $sSql = str_replace($sPattern, '', $sSql); - } - } - - return $sSql; - } -} diff --git a/lib-php/setup_functions.php b/lib-php/setup_functions.php index c89db534..179b0159 100755 --- a/lib-php/setup_functions.php +++ b/lib-php/setup_functions.php @@ -1,20 +1,5 @@ Date: Fri, 16 Apr 2021 18:41:14 +0200 Subject: [PATCH 3/3] remove transition functions from Python --- nominatim/cli.py | 2 - nominatim/clicmd/__init__.py | 1 - nominatim/clicmd/transition.py | 146 --------------------------------- 3 files changed, 149 deletions(-) delete mode 100644 nominatim/clicmd/transition.py diff --git a/nominatim/cli.py b/nominatim/cli.py index 9235055b..d560676e 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -273,8 +273,6 @@ def get_set_parser(**kwargs): else: parser.parser.epilog = 'php-cgi not found. Query commands not available.' - parser.add_subcommand('transition', clicmd.AdminTransition) - return parser diff --git a/nominatim/clicmd/__init__.py b/nominatim/clicmd/__init__.py index 9856ad68..f905fed1 100644 --- a/nominatim/clicmd/__init__.py +++ b/nominatim/clicmd/__init__.py @@ -9,5 +9,4 @@ from nominatim.clicmd.index import UpdateIndex from nominatim.clicmd.refresh import UpdateRefresh from nominatim.clicmd.admin import AdminFuncs from nominatim.clicmd.freeze import SetupFreeze -from nominatim.clicmd.transition import AdminTransition from nominatim.clicmd.special_phrases import ImportSpecialPhrases diff --git a/nominatim/clicmd/transition.py b/nominatim/clicmd/transition.py deleted file mode 100644 index f4df992c..00000000 --- a/nominatim/clicmd/transition.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -Implementation of the 'transition' subcommand. - -This subcommand provides standins for functions that were available -through the PHP scripts but are now no longer directly accessible. -This module will be removed as soon as the transition phase is over. -""" -import logging -from pathlib import Path - -from nominatim.db.connection import connect -from nominatim.db import status -from nominatim.errors import UsageError - -# Do not repeat documentation of subcommand classes. -# pylint: disable=C0111 -# Using non-top-level imports to avoid eventually unused imports. -# pylint: disable=E0012,C0415 - -LOG = logging.getLogger() - -class AdminTransition: - """\ - Internal functions for code transition. Do not use. - """ - - @staticmethod - def add_args(parser): - group = parser.add_argument_group('Sub-functions') - group.add_argument('--create-db', action='store_true', - help='Create nominatim db') - group.add_argument('--setup-db', action='store_true', - help='Build a blank nominatim db') - group.add_argument('--import-data', action='store_true', - help='Import a osm file') - group.add_argument('--load-data', action='store_true', - help='Copy data to live tables from import table') - group.add_argument('--create-tables', action='store_true', - help='Create main tables') - group.add_argument('--create-partition-tables', action='store_true', - help='Create required partition tables') - group.add_argument('--index', action='store_true', - help='Index the data') - group.add_argument('--create-search-indices', action='store_true', - help='Create additional indices required for search and update') - group.add_argument('--create-country-names', action='store_true', - help='Create search index for default country names.') - group = parser.add_argument_group('Options') - group.add_argument('--no-partitions', action='store_true', - help='Do not partition search indices') - group.add_argument('--osm-file', metavar='FILE', - help='File to import') - group.add_argument('--drop', action='store_true', - help='Drop tables needed for updates, making the database readonly') - group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int, - help='Size of cache to be used by osm2pgsql (in MB)') - group.add_argument('--no-analyse', action='store_true', - help='Do not perform analyse operations during index') - group.add_argument('--ignore-errors', action='store_true', - help="Ignore certain erros on import.") - group.add_argument('--reverse-only', action='store_true', - help='Do not create search tables and indexes') - group.add_argument('--tiger-data', metavar='FILE', - help='File to import') - - @staticmethod - def run(args): # pylint: disable=too-many-statements - from ..tools import database_import, tiger_data - from ..tools import refresh - - if args.create_db: - LOG.warning('Create DB') - database_import.create_db(args.config.get_libpq_dsn()) - - if args.setup_db: - LOG.warning('Setup DB') - - with connect(args.config.get_libpq_dsn()) as conn: - database_import.setup_extensions(conn) - database_import.install_module(args.module_dir, args.project_dir, - args.config.DATABASE_MODULE_PATH, - conn=conn) - - database_import.import_base_data(args.config.get_libpq_dsn(), - args.data_dir, args.no_partitions) - - if args.import_data: - LOG.warning('Import data') - if not args.osm_file: - raise UsageError('Missing required --osm-file argument') - database_import.import_osm_data(Path(args.osm_file), - args.osm2pgsql_options(0, 1), - drop=args.drop, - ignore_errors=args.ignore_errors) - - if args.create_tables: - LOG.warning('Create Tables') - with connect(args.config.get_libpq_dsn()) as conn: - database_import.create_tables(conn, args.config, args.sqllib_dir, args.reverse_only) - refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG)) - refresh.create_functions(conn, args.config, args.sqllib_dir, - enable_diff_updates=False) - database_import.create_table_triggers(conn, args.config, args.sqllib_dir) - - if args.create_partition_tables: - LOG.warning('Create Partition Tables') - with connect(args.config.get_libpq_dsn()) as conn: - database_import.create_partition_tables(conn, args.config, args.sqllib_dir) - - if args.load_data: - LOG.warning('Load data') - with connect(args.config.get_libpq_dsn()) as conn: - database_import.truncate_data_tables(conn, args.config.MAX_WORD_FREQUENCY) - database_import.load_data(args.config.get_libpq_dsn(), - args.data_dir, - args.threads or 1) - - with connect(args.config.get_libpq_dsn()) as conn: - try: - status.set_status(conn, status.compute_database_date(conn)) - except Exception as exc: # pylint: disable=broad-except - LOG.error('Cannot determine date of database: %s', exc) - - if args.index: - LOG.warning('Indexing') - from ..indexer.indexer import Indexer - indexer = Indexer(args.config.get_libpq_dsn(), args.threads or 1) - indexer.index_full() - - if args.create_search_indices: - LOG.warning('Create Search indices') - with connect(args.config.get_libpq_dsn()) as conn: - database_import.create_search_indices(conn, args.config, args.sqllib_dir, args.drop) - - if args.tiger_data: - LOG.warning('Tiger data') - tiger_data.add_tiger_data(args.config.get_libpq_dsn(), - args.tiger_data, - args.threads or 1, - args.config, - args.sqllib_dir) - - if args.create_country_names: - LOG.warning('Create search index for default country names.') - with connect(args.config.get_libpq_dsn()) as conn: - database_import.create_country_names(conn, args.config)