osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-09-11 19:17:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

remove old nominatim.py in favour of 'nominatim index'

Browse Source 
The PHP scripts need to know the position of the nominatim
tool in order to call it. This is handed in as environment
variable, so it can be set by the Python script.
This commit is contained in:
Sarah Hoffmann 2021-01-17 21:02:50 +01:00
parent c77877a934
commit 504922ffbe
10 changed files with 32 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmake/script.tmpl
View File

@ -8,5 +8,6 @@ require('@CMAKE_SOURCE_DIR@/lib/dotenv_loader.php');
@define('CONST_DataDir', '@CMAKE_SOURCE_DIR@');
loadDotEnv();
$_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim';
require_once('@CMAKE_SOURCE_DIR@/lib/admin/@script_source@');

3
cmake/tool.tmpl
View File

@ -1,8 +1,11 @@
#!/usr/bin/env python3
import sys
import os
sys.path.insert(1, '@CMAKE_SOURCE_DIR@')
os.environ['NOMINATIM_NOMINATIM_TOOL'] = __file__
from nominatim import cli
exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module',

2
lib/Shell.php
View File

@ -7,7 +7,7 @@ class Shell
public function __construct($sBaseCmd, ...$aParams)
{
if (!$sBaseCmd) {
throw new Exception('Command missing in new() call');
throw new \Exception('Command missing in new() call');
}
$this->baseCmd = $sBaseCmd;
$this->aParams = array();

37
lib/admin/update.php
View File

@ -105,25 +105,14 @@ if ($fPostgresVersion >= 11.0) {
}
$oIndexCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py'))
->addParams('--database', $aDSNInfo['database'])
->addParams('--port', $aDSNInfo['port'])
->addParams('--threads', $aResult['index-instances']);
if (!$aResult['quiet']) {
$oIndexCmd->addParams('--verbose');
$oIndexCmd = (new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
->addParams('index');
if ($aResult['quiet']) {
$oIndexCmd->addParams('--quiet');
}
if ($aResult['verbose']) {
$oIndexCmd->addParams('--verbose');
}
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
$oIndexCmd->addParams('--host', $aDSNInfo['hostspec']);
}
if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
$oIndexCmd->addParams('--username', $aDSNInfo['username']);
}
if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
$oIndexCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
}
$sPyosmiumBin = getSetting('PYOSMIUM_BINARY');
$sBaseURL = getSetting('REPLICATION_URL');
@ -288,15 +277,9 @@ if ($aResult['recompute-word-counts']) {
}
if ($aResult['index']) {
$oCmd = (clone $oIndexCmd)
->addParams('--minrank', $aResult['index-rank'], '-b');
$oCmd->run();
$oCmd = (clone $oIndexCmd)
->addParams('--minrank', $aResult['index-rank']);
$oCmd->run();
$oDB->exec('update import_status set indexed = true');
}
if ($aResult['update-address-levels']) {
@ -438,15 +421,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
if (!$aResult['no-index']) {
$fCMDStartTime = time();
$oThisIndexCmd = clone($oIndexCmd);
$oThisIndexCmd->addParams('-b');
echo $oThisIndexCmd->escapedCmd()."\n";
$iErrorLevel = $oThisIndexCmd->run();
if ($iErrorLevel) {
echo "Error: $iErrorLevel\n";
exit($iErrorLevel);
}
$oThisIndexCmd = clone($oIndexCmd);
echo $oThisIndexCmd->escapedCmd()."\n";
$iErrorLevel = $oThisIndexCmd->run();
@ -463,9 +437,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
var_Dump($sSQL);
$oDB->exec($sSQL);
echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
$sSQL = 'update import_status set indexed = true';
$oDB->exec($sSQL);
} else {
if ($aResult['import-osmosis-all']) {
echo "Error: --no-index cannot be used with continuous imports (--import-osmosis-all).\n";

25
lib/setup/SetupClass.php
View File

@ -549,26 +549,15 @@ class SetupFunctions
{
$this->checkModulePresence(); // raises exception on failure
$oBaseCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py'))
->addParams('--database', $this->aDSNInfo['database'])
->addParams('--port', $this->aDSNInfo['port'])
->addParams('--threads', $this->iInstances);
$oBaseCmd = (new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
->addParams('index');
if (!$this->bQuiet) {
$oBaseCmd->addParams('-v');
if ($this->bQuiet) {
$oBaseCmd->addParams('-q');
}
if ($this->bVerbose) {
$oBaseCmd->addParams('-v');
}
if (isset($this->aDSNInfo['hostspec'])) {
$oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']);
}
if (isset($this->aDSNInfo['username'])) {
$oBaseCmd->addParams('--user', $this->aDSNInfo['username']);
}
if (isset($this->aDSNInfo['password'])) {
$oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
}
info('Index ranks 0 - 4');
$oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
@ -581,14 +570,14 @@ class SetupFunctions
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index administrative boundaries');
$oCmd = (clone $oBaseCmd)->addParams('-b');
$oCmd = (clone $oBaseCmd)->addParams('--boundaries-only');
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
info('Index ranks 5 - 25');
$oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25);
$oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 5, '--maxrank', 25);
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
@ -597,7 +586,7 @@ class SetupFunctions
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index ranks 26 - 30');
$oCmd = (clone $oBaseCmd)->addParams('--minrank', 26);
$oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 26);
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');

5
nominatim/cli.py
View File

@ -78,7 +78,7 @@ class CommandlineParser:
args.project_dir = Path(args.project_dir)
logging.basicConfig(stream=sys.stderr,
format='%(asctime)s %(levelname)s: %(message)s',
format='%(asctime)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=max(4 - args.verbose, 1) * 10)
@ -328,6 +328,9 @@ class UpdateIndex:
if not args.boundaries_only:
indexer.index_by_rank(args.minrank, args.maxrank)
if not args.no_boundaries and not args.boundaries_only:
indexer.update_status_table()
return 0

7
nominatim/indexer/indexer.py
View File

@ -124,6 +124,13 @@ class Indexer:
else:
self.index(RankRunner(maxrank))
def update_status_table(self):
""" Update the status in the status table to 'indexed'.
"""
with self.conn.cursor() as cur:
cur.execute('UPDATE import_status SET indexed = true')
self.conn.commit()
def index(self, obj, batch=1):
""" Index a single rank or table. `obj` describes the SQL to use
for indexing. `batch` describes the number of objects that

8
nominatim/indexer/progress.py
View File

@ -26,7 +26,7 @@ class ProgressLogger:
self.done_places = 0
self.rank_start_time = datetime.now()
self.log_interval = log_interval
self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.INFO) else total + 1
self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.WARNING) else total + 1
def add(self, num=1):
""" Mark `num` places as processed. Print a log message if the
@ -47,9 +47,9 @@ class ProgressLogger:
places_per_sec = self.done_places / done_time
eta = (self.total_places - self.done_places) / places_per_sec
LOG.info("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f",
self.done_places, int(done_time),
places_per_sec, self.name, eta)
LOG.warning("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f",
self.done_places, int(done_time),
places_per_sec, self.name, eta)
self.next_info += int(places_per_sec) * self.log_interval

84
nominatim/nominatim.py
View File

@ -1,84 +0,0 @@
#! /usr/bin/env python3
#-----------------------------------------------------------------------------
# nominatim - [description]
#-----------------------------------------------------------------------------
#
# Indexing tool for the Nominatim database.
#
# Based on C version by Brian Quinion
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#-----------------------------------------------------------------------------
from argparse import ArgumentParser, RawDescriptionHelpFormatter
import logging
import sys
import getpass
from indexer.indexer import Indexer
def nominatim_arg_parser():
""" Setup the command-line parser for the tool.
"""
parser = ArgumentParser(description="Indexing tool for Nominatim.",
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('-d', '--database',
dest='dbname', action='store', default='nominatim',
help='Name of the PostgreSQL database to connect to.')
parser.add_argument('-U', '--username',
dest='user', action='store',
help='PostgreSQL user name.')
parser.add_argument('-W', '--password',
dest='password_prompt', action='store_true',
help='Force password prompt.')
parser.add_argument('-H', '--host',
dest='host', action='store',
help='PostgreSQL server hostname or socket location.')
parser.add_argument('-P', '--port',
dest='port', action='store',
help='PostgreSQL server port')
parser.add_argument('-b', '--boundary-only',
dest='boundary_only', action='store_true',
help='Only index administrative boundaries (ignores min/maxrank).')
parser.add_argument('-r', '--minrank',
dest='minrank', type=int, metavar='RANK', default=0,
help='Minimum/starting rank.')
parser.add_argument('-R', '--maxrank',
dest='maxrank', type=int, metavar='RANK', default=30,
help='Maximum/finishing rank.')
parser.add_argument('-t', '--threads',
dest='threads', type=int, metavar='NUM', default=1,
help='Number of threads to create for indexing.')
parser.add_argument('-v', '--verbose',
dest='loglevel', action='count', default=0,
help='Increase verbosity')
return parser
if __name__ == '__main__':
OPTIONS = nominatim_arg_parser().parse_args(sys.argv[1:])
logging.basicConfig(stream=sys.stderr, format='%(levelname)s: %(message)s',
level=max(3 - OPTIONS.loglevel, 0) * 10)
OPTIONS.password = None
if OPTIONS.password_prompt:
PASSWORD = getpass.getpass("Database password: ")
OPTIONS.password = PASSWORD
if OPTIONS.boundary_only:
Indexer(OPTIONS).index_boundaries()
else:
Indexer(OPTIONS).index_by_rank()

1
test/bdd/steps/nominatim_environment.py
View File

@ -91,6 +91,7 @@ class NominatimEnvironment:
self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils'
self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module'
self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql'
self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim'
if self.server_module_path:
self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path